1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2016 Nexenta Systems, Inc. All rights reserved. 14 * Copyright 2016 Tegile Systems, Inc. All rights reserved. 15 * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 16 */ 17 18 /* 19 * blkdev driver for NVMe compliant storage devices 20 * 21 * This driver was written to conform to version 1.0e of the NVMe specification. 22 * It may work with newer versions, but that is completely untested and disabled 23 * by default. 24 * 25 * The driver has only been tested on x86 systems and will not work on big- 26 * endian systems without changes to the code accessing registers and data 27 * structures used by the hardware. 28 * 29 * 30 * Interrupt Usage: 31 * 32 * The driver will use a FIXED interrupt while configuring the device as the 33 * specification requires. Later in the attach process it will switch to MSI-X 34 * or MSI if supported. The driver wants to have one interrupt vector per CPU, 35 * but it will work correctly if less are available. Interrupts can be shared 36 * by queues, the interrupt handler will iterate through the I/O queue array by 37 * steps of n_intr_cnt. Usually only the admin queue will share an interrupt 38 * with one I/O queue. The interrupt handler will retrieve completed commands 39 * from all queues sharing an interrupt vector and will post them to a taskq 40 * for completion processing. 41 * 42 * 43 * Command Processing: 44 * 45 * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up 46 * to 65536 I/O commands. The driver will configure one I/O queue pair per 47 * available interrupt vector, with the queue length usually much smaller than 48 * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 49 * interrupt vectors will be used. 50 * 51 * Additionally the hardware provides a single special admin queue pair that can 52 * hold up to 4096 admin commands. 53 * 54 * From the hardware perspective both queues of a queue pair are independent, 55 * but they share some driver state: the command array (holding pointers to 56 * commands currently being processed by the hardware) and the active command 57 * counter. Access to the submission side of a queue pair and the shared state 58 * is protected by nq_mutex. The completion side of a queue pair does not need 59 * that protection apart from its access to the shared state; it is called only 60 * in the interrupt handler which does not run concurrently for the same 61 * interrupt vector. 62 * 63 * When a command is submitted to a queue pair the active command counter is 64 * incremented and a pointer to the command is stored in the command array. The 65 * array index is used as command identifier (CID) in the submission queue 66 * entry. Some commands may take a very long time to complete, and if the queue 67 * wraps around in that time a submission may find the next array slot to still 68 * be used by a long-running command. In this case the array is sequentially 69 * searched for the next free slot. The length of the command array is the same 70 * as the configured queue length. 71 * 72 * 73 * Namespace Support: 74 * 75 * NVMe devices can have multiple namespaces, each being a independent data 76 * store. The driver supports multiple namespaces and creates a blkdev interface 77 * for each namespace found. Namespaces can have various attributes to support 78 * thin provisioning, extended LBAs, and protection information. This driver 79 * does not support any of this and ignores namespaces that have these 80 * attributes. 81 * 82 * 83 * Blkdev Interface: 84 * 85 * This driver uses blkdev to do all the heavy lifting involved with presenting 86 * a disk device to the system. As a result, the processing of I/O requests is 87 * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 88 * setup, and splitting of transfers into manageable chunks. 89 * 90 * I/O requests coming in from blkdev are turned into NVM commands and posted to 91 * an I/O queue. The queue is selected by taking the CPU id modulo the number of 92 * queues. There is currently no timeout handling of I/O commands. 93 * 94 * Blkdev also supports querying device/media information and generating a 95 * devid. The driver reports the best block size as determined by the namespace 96 * format back to blkdev as physical block size to support partition and block 97 * alignment. The devid is composed using the device vendor ID, model number, 98 * serial number, and the namespace ID. 99 * 100 * 101 * Error Handling: 102 * 103 * Error handling is currently limited to detecting fatal hardware errors, 104 * either by asynchronous events, or synchronously through command status or 105 * admin command timeouts. In case of severe errors the device is fenced off, 106 * all further requests will return EIO. FMA is then called to fault the device. 107 * 108 * The hardware has a limit for outstanding asynchronous event requests. Before 109 * this limit is known the driver assumes it is at least 1 and posts a single 110 * asynchronous request. Later when the limit is known more asynchronous event 111 * requests are posted to allow quicker reception of error information. When an 112 * asynchronous event is posted by the hardware the driver will parse the error 113 * status fields and log information or fault the device, depending on the 114 * severity of the asynchronous event. The asynchronous event request is then 115 * reused and posted to the admin queue again. 116 * 117 * On command completion the command status is checked for errors. In case of 118 * errors indicating a driver bug the driver panics. Almost all other error 119 * status values just cause EIO to be returned. 120 * 121 * Command timeouts are currently detected for all admin commands except 122 * asynchronous event requests. If a command times out and the hardware appears 123 * to be healthy the driver attempts to abort the command. If this fails the 124 * driver assumes the device to be dead, fences it off, and calls FMA to retire 125 * it. In general admin commands are issued at attach time only. No timeout 126 * handling of normal I/O commands is presently done. 127 * 128 * In some cases it may be possible that the ABORT command times out, too. In 129 * that case the device is also declared dead and fenced off. 130 * 131 * 132 * Quiesce / Fast Reboot: 133 * 134 * The driver currently does not support fast reboot. A quiesce(9E) entry point 135 * is still provided which is used to send a shutdown notification to the 136 * device. 137 * 138 * 139 * Driver Configuration: 140 * 141 * The following driver properties can be changed to control some aspects of the 142 * drivers operation: 143 * - strict-version: can be set to 0 to allow devices conforming to newer 144 * versions to be used 145 * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 146 * specific command status as a fatal error leading device faulting 147 * - admin-queue-len: the maximum length of the admin queue (16-4096) 148 * - io-queue-len: the maximum length of the I/O queues (16-65536) 149 * - async-event-limit: the maximum number of asynchronous event requests to be 150 * posted by the driver 151 * 152 * 153 * TODO: 154 * - figure out sane default for I/O queue depth reported to blkdev 155 * - polled I/O support to support kernel core dumping 156 * - FMA handling of media errors 157 * - support for the Volatile Write Cache 158 * - support for devices supporting very large I/O requests using chained PRPs 159 * - support for querying log pages from user space 160 * - support for configuring hardware parameters like interrupt coalescing 161 * - support for media formatting and hard partitioning into namespaces 162 * - support for big-endian systems 163 * - support for fast reboot 164 */ 165 166 #include <sys/byteorder.h> 167 #ifdef _BIG_ENDIAN 168 #error nvme driver needs porting for big-endian platforms 169 #endif 170 171 #include <sys/modctl.h> 172 #include <sys/conf.h> 173 #include <sys/devops.h> 174 #include <sys/ddi.h> 175 #include <sys/sunddi.h> 176 #include <sys/bitmap.h> 177 #include <sys/sysmacros.h> 178 #include <sys/param.h> 179 #include <sys/varargs.h> 180 #include <sys/cpuvar.h> 181 #include <sys/disp.h> 182 #include <sys/blkdev.h> 183 #include <sys/atomic.h> 184 #include <sys/archsystm.h> 185 #include <sys/sata/sata_hba.h> 186 187 #include "nvme_reg.h" 188 #include "nvme_var.h" 189 190 191 /* NVMe spec version supported */ 192 static const int nvme_version_major = 1; 193 static const int nvme_version_minor = 0; 194 195 /* tunable for admin command timeout in seconds, default is 1s */ 196 static volatile int nvme_admin_cmd_timeout = 1; 197 198 static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 199 static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 200 static int nvme_quiesce(dev_info_t *); 201 static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 202 static int nvme_setup_interrupts(nvme_t *, int, int); 203 static void nvme_release_interrupts(nvme_t *); 204 static uint_t nvme_intr(caddr_t, caddr_t); 205 206 static void nvme_shutdown(nvme_t *, int, boolean_t); 207 static boolean_t nvme_reset(nvme_t *, boolean_t); 208 static int nvme_init(nvme_t *); 209 static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 210 static void nvme_free_cmd(nvme_cmd_t *); 211 static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 212 bd_xfer_t *); 213 static int nvme_admin_cmd(nvme_cmd_t *, int); 214 static int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *); 215 static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 216 static boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t); 217 static void nvme_wakeup_cmd(void *); 218 static void nvme_async_event_task(void *); 219 220 static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 221 static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 222 static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 223 static int nvme_check_specific_cmd_status(nvme_cmd_t *); 224 static int nvme_check_generic_cmd_status(nvme_cmd_t *); 225 static inline int nvme_check_cmd_status(nvme_cmd_t *); 226 227 static void nvme_abort_cmd(nvme_cmd_t *); 228 static int nvme_async_event(nvme_t *); 229 static void *nvme_get_logpage(nvme_t *, uint8_t, ...); 230 static void *nvme_identify(nvme_t *, uint32_t); 231 static int nvme_set_nqueues(nvme_t *, uint16_t); 232 233 static void nvme_free_dma(nvme_dma_t *); 234 static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 235 nvme_dma_t **); 236 static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 237 nvme_dma_t **); 238 static void nvme_free_qpair(nvme_qpair_t *); 239 static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int); 240 static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 241 242 static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 243 static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 244 static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 245 static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 246 247 static boolean_t nvme_check_regs_hdl(nvme_t *); 248 static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 249 250 static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 251 252 static void nvme_bd_xfer_done(void *); 253 static void nvme_bd_driveinfo(void *, bd_drive_t *); 254 static int nvme_bd_mediainfo(void *, bd_media_t *); 255 static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 256 static int nvme_bd_read(void *, bd_xfer_t *); 257 static int nvme_bd_write(void *, bd_xfer_t *); 258 static int nvme_bd_sync(void *, bd_xfer_t *); 259 static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 260 261 static void nvme_prepare_devid(nvme_t *, uint32_t); 262 263 static void *nvme_state; 264 static kmem_cache_t *nvme_cmd_cache; 265 266 /* 267 * DMA attributes for queue DMA memory 268 * 269 * Queue DMA memory must be page aligned. The maximum length of a queue is 270 * 65536 entries, and an entry can be 64 bytes long. 271 */ 272 static ddi_dma_attr_t nvme_queue_dma_attr = { 273 .dma_attr_version = DMA_ATTR_V0, 274 .dma_attr_addr_lo = 0, 275 .dma_attr_addr_hi = 0xffffffffffffffffULL, 276 .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 277 .dma_attr_align = 0x1000, 278 .dma_attr_burstsizes = 0x7ff, 279 .dma_attr_minxfer = 0x1000, 280 .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 281 .dma_attr_seg = 0xffffffffffffffffULL, 282 .dma_attr_sgllen = 1, 283 .dma_attr_granular = 1, 284 .dma_attr_flags = 0, 285 }; 286 287 /* 288 * DMA attributes for transfers using Physical Region Page (PRP) entries 289 * 290 * A PRP entry describes one page of DMA memory using the page size specified 291 * in the controller configuration's memory page size register (CC.MPS). It uses 292 * a 64bit base address aligned to this page size. There is no limitation on 293 * chaining PRPs together for arbitrarily large DMA transfers. 294 */ 295 static ddi_dma_attr_t nvme_prp_dma_attr = { 296 .dma_attr_version = DMA_ATTR_V0, 297 .dma_attr_addr_lo = 0, 298 .dma_attr_addr_hi = 0xffffffffffffffffULL, 299 .dma_attr_count_max = 0xfff, 300 .dma_attr_align = 0x1000, 301 .dma_attr_burstsizes = 0x7ff, 302 .dma_attr_minxfer = 0x1000, 303 .dma_attr_maxxfer = 0x1000, 304 .dma_attr_seg = 0xfff, 305 .dma_attr_sgllen = -1, 306 .dma_attr_granular = 1, 307 .dma_attr_flags = 0, 308 }; 309 310 /* 311 * DMA attributes for transfers using scatter/gather lists 312 * 313 * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 314 * 32bit length field. SGL Segment and SGL Last Segment entries require the 315 * length to be a multiple of 16 bytes. 316 */ 317 static ddi_dma_attr_t nvme_sgl_dma_attr = { 318 .dma_attr_version = DMA_ATTR_V0, 319 .dma_attr_addr_lo = 0, 320 .dma_attr_addr_hi = 0xffffffffffffffffULL, 321 .dma_attr_count_max = 0xffffffffUL, 322 .dma_attr_align = 1, 323 .dma_attr_burstsizes = 0x7ff, 324 .dma_attr_minxfer = 0x10, 325 .dma_attr_maxxfer = 0xfffffffffULL, 326 .dma_attr_seg = 0xffffffffffffffffULL, 327 .dma_attr_sgllen = -1, 328 .dma_attr_granular = 0x10, 329 .dma_attr_flags = 0 330 }; 331 332 static ddi_device_acc_attr_t nvme_reg_acc_attr = { 333 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 334 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 335 .devacc_attr_dataorder = DDI_STRICTORDER_ACC 336 }; 337 338 static struct dev_ops nvme_dev_ops = { 339 .devo_rev = DEVO_REV, 340 .devo_refcnt = 0, 341 .devo_getinfo = ddi_no_info, 342 .devo_identify = nulldev, 343 .devo_probe = nulldev, 344 .devo_attach = nvme_attach, 345 .devo_detach = nvme_detach, 346 .devo_reset = nodev, 347 .devo_cb_ops = NULL, 348 .devo_bus_ops = NULL, 349 .devo_power = NULL, 350 .devo_quiesce = nvme_quiesce, 351 }; 352 353 static struct modldrv nvme_modldrv = { 354 .drv_modops = &mod_driverops, 355 .drv_linkinfo = "NVMe v1.0e", 356 .drv_dev_ops = &nvme_dev_ops 357 }; 358 359 static struct modlinkage nvme_modlinkage = { 360 .ml_rev = MODREV_1, 361 .ml_linkage = { &nvme_modldrv, NULL } 362 }; 363 364 static bd_ops_t nvme_bd_ops = { 365 .o_version = BD_OPS_VERSION_0, 366 .o_drive_info = nvme_bd_driveinfo, 367 .o_media_info = nvme_bd_mediainfo, 368 .o_devid_init = nvme_bd_devid, 369 .o_sync_cache = nvme_bd_sync, 370 .o_read = nvme_bd_read, 371 .o_write = nvme_bd_write, 372 }; 373 374 int 375 _init(void) 376 { 377 int error; 378 379 error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 380 if (error != DDI_SUCCESS) 381 return (error); 382 383 nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 384 sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 385 386 bd_mod_init(&nvme_dev_ops); 387 388 error = mod_install(&nvme_modlinkage); 389 if (error != DDI_SUCCESS) { 390 ddi_soft_state_fini(&nvme_state); 391 bd_mod_fini(&nvme_dev_ops); 392 } 393 394 return (error); 395 } 396 397 int 398 _fini(void) 399 { 400 int error; 401 402 error = mod_remove(&nvme_modlinkage); 403 if (error == DDI_SUCCESS) { 404 ddi_soft_state_fini(&nvme_state); 405 kmem_cache_destroy(nvme_cmd_cache); 406 bd_mod_fini(&nvme_dev_ops); 407 } 408 409 return (error); 410 } 411 412 int 413 _info(struct modinfo *modinfop) 414 { 415 return (mod_info(&nvme_modlinkage, modinfop)); 416 } 417 418 static inline void 419 nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 420 { 421 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 422 423 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 424 ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 425 } 426 427 static inline void 428 nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 429 { 430 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 431 432 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 433 ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 434 } 435 436 static inline uint64_t 437 nvme_get64(nvme_t *nvme, uintptr_t reg) 438 { 439 uint64_t val; 440 441 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 442 443 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 444 val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 445 446 return (val); 447 } 448 449 static inline uint32_t 450 nvme_get32(nvme_t *nvme, uintptr_t reg) 451 { 452 uint32_t val; 453 454 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 455 456 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 457 val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 458 459 return (val); 460 } 461 462 static boolean_t 463 nvme_check_regs_hdl(nvme_t *nvme) 464 { 465 ddi_fm_error_t error; 466 467 ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 468 469 if (error.fme_status != DDI_FM_OK) 470 return (B_TRUE); 471 472 return (B_FALSE); 473 } 474 475 static boolean_t 476 nvme_check_dma_hdl(nvme_dma_t *dma) 477 { 478 ddi_fm_error_t error; 479 480 if (dma == NULL) 481 return (B_FALSE); 482 483 ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 484 485 if (error.fme_status != DDI_FM_OK) 486 return (B_TRUE); 487 488 return (B_FALSE); 489 } 490 491 static void 492 nvme_free_dma(nvme_dma_t *dma) 493 { 494 if (dma->nd_dmah != NULL) 495 (void) ddi_dma_unbind_handle(dma->nd_dmah); 496 if (dma->nd_acch != NULL) 497 ddi_dma_mem_free(&dma->nd_acch); 498 if (dma->nd_dmah != NULL) 499 ddi_dma_free_handle(&dma->nd_dmah); 500 kmem_free(dma, sizeof (nvme_dma_t)); 501 } 502 503 static int 504 nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 505 ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 506 { 507 nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 508 509 if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 510 &dma->nd_dmah) != DDI_SUCCESS) { 511 /* 512 * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 513 * the only other possible error is DDI_DMA_BADATTR which 514 * indicates a driver bug which should cause a panic. 515 */ 516 dev_err(nvme->n_dip, CE_PANIC, 517 "!failed to get DMA handle, check DMA attributes"); 518 return (DDI_FAILURE); 519 } 520 521 /* 522 * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 523 * or the flags are conflicting, which isn't the case here. 524 */ 525 (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 526 DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 527 &dma->nd_len, &dma->nd_acch); 528 529 if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 530 dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 531 &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 532 dev_err(nvme->n_dip, CE_WARN, 533 "!failed to bind DMA memory"); 534 atomic_inc_32(&nvme->n_dma_bind_err); 535 *ret = NULL; 536 nvme_free_dma(dma); 537 return (DDI_FAILURE); 538 } 539 540 bzero(dma->nd_memp, dma->nd_len); 541 542 *ret = dma; 543 return (DDI_SUCCESS); 544 } 545 546 static int 547 nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 548 uint_t flags, nvme_dma_t **dma) 549 { 550 uint32_t len = nentry * qe_len; 551 ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 552 553 len = roundup(len, nvme->n_pagesize); 554 555 q_dma_attr.dma_attr_minxfer = len; 556 557 if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 558 != DDI_SUCCESS) { 559 dev_err(nvme->n_dip, CE_WARN, 560 "!failed to get DMA memory for queue"); 561 goto fail; 562 } 563 564 if ((*dma)->nd_ncookie != 1) { 565 dev_err(nvme->n_dip, CE_WARN, 566 "!got too many cookies for queue DMA"); 567 goto fail; 568 } 569 570 return (DDI_SUCCESS); 571 572 fail: 573 if (*dma) { 574 nvme_free_dma(*dma); 575 *dma = NULL; 576 } 577 578 return (DDI_FAILURE); 579 } 580 581 static void 582 nvme_free_qpair(nvme_qpair_t *qp) 583 { 584 int i; 585 586 mutex_destroy(&qp->nq_mutex); 587 588 if (qp->nq_sqdma != NULL) 589 nvme_free_dma(qp->nq_sqdma); 590 if (qp->nq_cqdma != NULL) 591 nvme_free_dma(qp->nq_cqdma); 592 593 if (qp->nq_active_cmds > 0) 594 for (i = 0; i != qp->nq_nentry; i++) 595 if (qp->nq_cmd[i] != NULL) 596 nvme_free_cmd(qp->nq_cmd[i]); 597 598 if (qp->nq_cmd != NULL) 599 kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 600 601 kmem_free(qp, sizeof (nvme_qpair_t)); 602 } 603 604 static int 605 nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 606 int idx) 607 { 608 nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 609 610 mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 611 DDI_INTR_PRI(nvme->n_intr_pri)); 612 613 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 614 DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 615 goto fail; 616 617 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 618 DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS) 619 goto fail; 620 621 qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 622 qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp; 623 qp->nq_nentry = nentry; 624 625 qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 626 qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx); 627 628 qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 629 qp->nq_next_cmd = 0; 630 631 *nqp = qp; 632 return (DDI_SUCCESS); 633 634 fail: 635 nvme_free_qpair(qp); 636 *nqp = NULL; 637 638 return (DDI_FAILURE); 639 } 640 641 static nvme_cmd_t * 642 nvme_alloc_cmd(nvme_t *nvme, int kmflag) 643 { 644 nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 645 646 if (cmd == NULL) 647 return (cmd); 648 649 bzero(cmd, sizeof (nvme_cmd_t)); 650 651 cmd->nc_nvme = nvme; 652 653 mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 654 DDI_INTR_PRI(nvme->n_intr_pri)); 655 cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 656 657 return (cmd); 658 } 659 660 static void 661 nvme_free_cmd(nvme_cmd_t *cmd) 662 { 663 if (cmd->nc_dma) { 664 nvme_free_dma(cmd->nc_dma); 665 cmd->nc_dma = NULL; 666 } 667 668 cv_destroy(&cmd->nc_cv); 669 mutex_destroy(&cmd->nc_mutex); 670 671 kmem_cache_free(nvme_cmd_cache, cmd); 672 } 673 674 static int 675 nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 676 { 677 nvme_reg_sqtdbl_t tail = { 0 }; 678 679 mutex_enter(&qp->nq_mutex); 680 681 if (qp->nq_active_cmds == qp->nq_nentry) { 682 mutex_exit(&qp->nq_mutex); 683 return (DDI_FAILURE); 684 } 685 686 cmd->nc_completed = B_FALSE; 687 688 /* 689 * Try to insert the cmd into the active cmd array at the nq_next_cmd 690 * slot. If the slot is already occupied advance to the next slot and 691 * try again. This can happen for long running commands like async event 692 * requests. 693 */ 694 while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 695 qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 696 qp->nq_cmd[qp->nq_next_cmd] = cmd; 697 698 qp->nq_active_cmds++; 699 700 cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 701 bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 702 (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 703 sizeof (nvme_sqe_t) * qp->nq_sqtail, 704 sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 705 qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 706 707 tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 708 nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 709 710 mutex_exit(&qp->nq_mutex); 711 return (DDI_SUCCESS); 712 } 713 714 static nvme_cmd_t * 715 nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 716 { 717 nvme_reg_cqhdbl_t head = { 0 }; 718 719 nvme_cqe_t *cqe; 720 nvme_cmd_t *cmd; 721 722 (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0, 723 sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL); 724 725 cqe = &qp->nq_cq[qp->nq_cqhead]; 726 727 /* Check phase tag of CQE. Hardware inverts it for new entries. */ 728 if (cqe->cqe_sf.sf_p == qp->nq_phase) 729 return (NULL); 730 731 ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp); 732 ASSERT(cqe->cqe_cid < qp->nq_nentry); 733 734 mutex_enter(&qp->nq_mutex); 735 cmd = qp->nq_cmd[cqe->cqe_cid]; 736 qp->nq_cmd[cqe->cqe_cid] = NULL; 737 qp->nq_active_cmds--; 738 mutex_exit(&qp->nq_mutex); 739 740 ASSERT(cmd != NULL); 741 ASSERT(cmd->nc_nvme == nvme); 742 ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 743 ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid); 744 bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 745 746 qp->nq_sqhead = cqe->cqe_sqhd; 747 748 head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry; 749 750 /* Toggle phase on wrap-around. */ 751 if (qp->nq_cqhead == 0) 752 qp->nq_phase = qp->nq_phase ? 0 : 1; 753 754 nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); 755 756 return (cmd); 757 } 758 759 static int 760 nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 761 { 762 nvme_cqe_t *cqe = &cmd->nc_cqe; 763 764 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 765 "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 766 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 767 cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 768 cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 769 770 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 771 772 if (cmd->nc_nvme->n_strict_version) { 773 cmd->nc_nvme->n_dead = B_TRUE; 774 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 775 } 776 777 return (EIO); 778 } 779 780 static int 781 nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 782 { 783 nvme_cqe_t *cqe = &cmd->nc_cqe; 784 785 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 786 "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 787 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 788 cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 789 cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 790 if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 791 cmd->nc_nvme->n_dead = B_TRUE; 792 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 793 } 794 795 return (EIO); 796 } 797 798 static int 799 nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 800 { 801 nvme_cqe_t *cqe = &cmd->nc_cqe; 802 803 switch (cqe->cqe_sf.sf_sc) { 804 case NVME_CQE_SC_INT_NVM_WRITE: 805 /* write fail */ 806 /* TODO: post ereport */ 807 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 808 return (EIO); 809 810 case NVME_CQE_SC_INT_NVM_READ: 811 /* read fail */ 812 /* TODO: post ereport */ 813 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 814 return (EIO); 815 816 default: 817 return (nvme_check_unknown_cmd_status(cmd)); 818 } 819 } 820 821 static int 822 nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 823 { 824 nvme_cqe_t *cqe = &cmd->nc_cqe; 825 826 switch (cqe->cqe_sf.sf_sc) { 827 case NVME_CQE_SC_GEN_SUCCESS: 828 return (0); 829 830 /* 831 * Errors indicating a bug in the driver should cause a panic. 832 */ 833 case NVME_CQE_SC_GEN_INV_OPC: 834 /* Invalid Command Opcode */ 835 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 836 "invalid opcode in cmd %p", (void *)cmd); 837 return (0); 838 839 case NVME_CQE_SC_GEN_INV_FLD: 840 /* Invalid Field in Command */ 841 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 842 "invalid field in cmd %p", (void *)cmd); 843 return (0); 844 845 case NVME_CQE_SC_GEN_ID_CNFL: 846 /* Command ID Conflict */ 847 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 848 "cmd ID conflict in cmd %p", (void *)cmd); 849 return (0); 850 851 case NVME_CQE_SC_GEN_INV_NS: 852 /* Invalid Namespace or Format */ 853 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 854 "invalid NS/format in cmd %p", (void *)cmd); 855 return (0); 856 857 case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 858 /* LBA Out Of Range */ 859 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 860 "LBA out of range in cmd %p", (void *)cmd); 861 return (0); 862 863 /* 864 * Non-fatal errors, handle gracefully. 865 */ 866 case NVME_CQE_SC_GEN_DATA_XFR_ERR: 867 /* Data Transfer Error (DMA) */ 868 /* TODO: post ereport */ 869 atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 870 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 871 return (EIO); 872 873 case NVME_CQE_SC_GEN_INTERNAL_ERR: 874 /* 875 * Internal Error. The spec (v1.0, section 4.5.1.2) says 876 * detailed error information is returned as async event, 877 * so we pretty much ignore the error here and handle it 878 * in the async event handler. 879 */ 880 atomic_inc_32(&cmd->nc_nvme->n_internal_err); 881 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 882 return (EIO); 883 884 case NVME_CQE_SC_GEN_ABORT_REQUEST: 885 /* 886 * Command Abort Requested. This normally happens only when a 887 * command times out. 888 */ 889 /* TODO: post ereport or change blkdev to handle this? */ 890 atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 891 return (ECANCELED); 892 893 case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 894 /* Command Aborted due to Power Loss Notification */ 895 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 896 cmd->nc_nvme->n_dead = B_TRUE; 897 return (EIO); 898 899 case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 900 /* Command Aborted due to SQ Deletion */ 901 atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 902 return (EIO); 903 904 case NVME_CQE_SC_GEN_NVM_CAP_EXC: 905 /* Capacity Exceeded */ 906 atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 907 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 908 return (EIO); 909 910 case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 911 /* Namespace Not Ready */ 912 atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 913 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 914 return (EIO); 915 916 default: 917 return (nvme_check_unknown_cmd_status(cmd)); 918 } 919 } 920 921 static int 922 nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 923 { 924 nvme_cqe_t *cqe = &cmd->nc_cqe; 925 926 switch (cqe->cqe_sf.sf_sc) { 927 case NVME_CQE_SC_SPC_INV_CQ: 928 /* Completion Queue Invalid */ 929 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 930 atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 931 return (EINVAL); 932 933 case NVME_CQE_SC_SPC_INV_QID: 934 /* Invalid Queue Identifier */ 935 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 936 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 937 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 938 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 939 atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 940 return (EINVAL); 941 942 case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 943 /* Max Queue Size Exceeded */ 944 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 945 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 946 atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 947 return (EINVAL); 948 949 case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 950 /* Abort Command Limit Exceeded */ 951 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 952 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 953 "abort command limit exceeded in cmd %p", (void *)cmd); 954 return (0); 955 956 case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 957 /* Async Event Request Limit Exceeded */ 958 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 959 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 960 "async event request limit exceeded in cmd %p", 961 (void *)cmd); 962 return (0); 963 964 case NVME_CQE_SC_SPC_INV_INT_VECT: 965 /* Invalid Interrupt Vector */ 966 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 967 atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 968 return (EINVAL); 969 970 case NVME_CQE_SC_SPC_INV_LOG_PAGE: 971 /* Invalid Log Page */ 972 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 973 atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 974 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 975 return (EINVAL); 976 977 case NVME_CQE_SC_SPC_INV_FORMAT: 978 /* Invalid Format */ 979 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 980 atomic_inc_32(&cmd->nc_nvme->n_inv_format); 981 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 982 return (EINVAL); 983 984 case NVME_CQE_SC_SPC_INV_Q_DEL: 985 /* Invalid Queue Deletion */ 986 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 987 atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 988 return (EINVAL); 989 990 case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 991 /* Conflicting Attributes */ 992 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 993 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 994 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 995 atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 996 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 997 return (EINVAL); 998 999 case NVME_CQE_SC_SPC_NVM_INV_PROT: 1000 /* Invalid Protection Information */ 1001 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 1002 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 1003 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 1004 atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 1005 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1006 return (EINVAL); 1007 1008 case NVME_CQE_SC_SPC_NVM_READONLY: 1009 /* Write to Read Only Range */ 1010 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 1011 atomic_inc_32(&cmd->nc_nvme->n_readonly); 1012 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1013 return (EROFS); 1014 1015 default: 1016 return (nvme_check_unknown_cmd_status(cmd)); 1017 } 1018 } 1019 1020 static inline int 1021 nvme_check_cmd_status(nvme_cmd_t *cmd) 1022 { 1023 nvme_cqe_t *cqe = &cmd->nc_cqe; 1024 1025 /* take a shortcut if everything is alright */ 1026 if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 1027 cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 1028 return (0); 1029 1030 if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 1031 return (nvme_check_generic_cmd_status(cmd)); 1032 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 1033 return (nvme_check_specific_cmd_status(cmd)); 1034 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 1035 return (nvme_check_integrity_cmd_status(cmd)); 1036 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 1037 return (nvme_check_vendor_cmd_status(cmd)); 1038 1039 return (nvme_check_unknown_cmd_status(cmd)); 1040 } 1041 1042 /* 1043 * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands 1044 * 1045 * This functions takes care of cleaning up aborted commands. The command 1046 * status is checked to catch any fatal errors. 1047 */ 1048 static void 1049 nvme_abort_cmd_cb(void *arg) 1050 { 1051 nvme_cmd_t *cmd = arg; 1052 1053 /* 1054 * Grab the command mutex. Once we have it we hold the last reference 1055 * to the command and can safely free it. 1056 */ 1057 mutex_enter(&cmd->nc_mutex); 1058 (void) nvme_check_cmd_status(cmd); 1059 mutex_exit(&cmd->nc_mutex); 1060 1061 nvme_free_cmd(cmd); 1062 } 1063 1064 static void 1065 nvme_abort_cmd(nvme_cmd_t *abort_cmd) 1066 { 1067 nvme_t *nvme = abort_cmd->nc_nvme; 1068 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1069 nvme_abort_cmd_t ac = { 0 }; 1070 1071 sema_p(&nvme->n_abort_sema); 1072 1073 ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 1074 ac.b.ac_sqid = abort_cmd->nc_sqid; 1075 1076 /* 1077 * Drop the mutex of the aborted command. From this point on 1078 * we must assume that the abort callback has freed the command. 1079 */ 1080 mutex_exit(&abort_cmd->nc_mutex); 1081 1082 cmd->nc_sqid = 0; 1083 cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 1084 cmd->nc_callback = nvme_wakeup_cmd; 1085 cmd->nc_sqe.sqe_cdw10 = ac.r; 1086 1087 /* 1088 * Send the ABORT to the hardware. The ABORT command will return _after_ 1089 * the aborted command has completed (aborted or otherwise). 1090 */ 1091 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1092 sema_v(&nvme->n_abort_sema); 1093 dev_err(nvme->n_dip, CE_WARN, 1094 "!nvme_admin_cmd failed for ABORT"); 1095 atomic_inc_32(&nvme->n_abort_failed); 1096 return; 1097 } 1098 sema_v(&nvme->n_abort_sema); 1099 1100 if (nvme_check_cmd_status(cmd)) { 1101 dev_err(nvme->n_dip, CE_WARN, 1102 "!ABORT failed with sct = %x, sc = %x", 1103 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1104 atomic_inc_32(&nvme->n_abort_failed); 1105 } else { 1106 atomic_inc_32(&nvme->n_cmd_aborted); 1107 } 1108 1109 nvme_free_cmd(cmd); 1110 } 1111 1112 /* 1113 * nvme_wait_cmd -- wait for command completion or timeout 1114 * 1115 * Returns B_TRUE if the command completed normally. 1116 * 1117 * Returns B_FALSE if the command timed out and an abort was attempted. The 1118 * command mutex will be dropped and the command must be considered freed. The 1119 * freeing of the command is normally done by the abort command callback. 1120 * 1121 * In case of a serious error or a timeout of the abort command the hardware 1122 * will be declared dead and FMA will be notified. 1123 */ 1124 static boolean_t 1125 nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 1126 { 1127 clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 1128 nvme_t *nvme = cmd->nc_nvme; 1129 nvme_reg_csts_t csts; 1130 1131 ASSERT(mutex_owned(&cmd->nc_mutex)); 1132 1133 while (!cmd->nc_completed) { 1134 if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 1135 break; 1136 } 1137 1138 if (cmd->nc_completed) 1139 return (B_TRUE); 1140 1141 /* 1142 * The command timed out. Change the callback to the cleanup function. 1143 */ 1144 cmd->nc_callback = nvme_abort_cmd_cb; 1145 1146 /* 1147 * Check controller for fatal status, any errors associated with the 1148 * register or DMA handle, or for a double timeout (abort command timed 1149 * out). If necessary log a warning and call FMA. 1150 */ 1151 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1152 dev_err(nvme->n_dip, CE_WARN, "!command timeout, " 1153 "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 1154 atomic_inc_32(&nvme->n_cmd_timeout); 1155 1156 if (csts.b.csts_cfs || 1157 nvme_check_regs_hdl(nvme) || 1158 nvme_check_dma_hdl(cmd->nc_dma) || 1159 cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 1160 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1161 nvme->n_dead = B_TRUE; 1162 mutex_exit(&cmd->nc_mutex); 1163 } else { 1164 /* 1165 * Try to abort the command. The command mutex is released by 1166 * nvme_abort_cmd(). 1167 * If the abort succeeds it will have freed the aborted command. 1168 * If the abort fails for other reasons we must assume that the 1169 * command may complete at any time, and the callback will free 1170 * it for us. 1171 */ 1172 nvme_abort_cmd(cmd); 1173 } 1174 1175 return (B_FALSE); 1176 } 1177 1178 static void 1179 nvme_wakeup_cmd(void *arg) 1180 { 1181 nvme_cmd_t *cmd = arg; 1182 1183 mutex_enter(&cmd->nc_mutex); 1184 /* 1185 * There is a slight chance that this command completed shortly after 1186 * the timeout was hit in nvme_wait_cmd() but before the callback was 1187 * changed. Catch that case here and clean up accordingly. 1188 */ 1189 if (cmd->nc_callback == nvme_abort_cmd_cb) { 1190 mutex_exit(&cmd->nc_mutex); 1191 nvme_abort_cmd_cb(cmd); 1192 return; 1193 } 1194 1195 cmd->nc_completed = B_TRUE; 1196 cv_signal(&cmd->nc_cv); 1197 mutex_exit(&cmd->nc_mutex); 1198 } 1199 1200 static void 1201 nvme_async_event_task(void *arg) 1202 { 1203 nvme_cmd_t *cmd = arg; 1204 nvme_t *nvme = cmd->nc_nvme; 1205 nvme_error_log_entry_t *error_log = NULL; 1206 nvme_health_log_t *health_log = NULL; 1207 nvme_async_event_t event; 1208 int ret; 1209 1210 /* 1211 * Check for errors associated with the async request itself. The only 1212 * command-specific error is "async event limit exceeded", which 1213 * indicates a programming error in the driver and causes a panic in 1214 * nvme_check_cmd_status(). 1215 * 1216 * Other possible errors are various scenarios where the async request 1217 * was aborted, or internal errors in the device. Internal errors are 1218 * reported to FMA, the command aborts need no special handling here. 1219 */ 1220 if (nvme_check_cmd_status(cmd)) { 1221 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 1222 "!async event request returned failure, sct = %x, " 1223 "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 1224 cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 1225 cmd->nc_cqe.cqe_sf.sf_m); 1226 1227 if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 1228 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 1229 cmd->nc_nvme->n_dead = B_TRUE; 1230 ddi_fm_service_impact(cmd->nc_nvme->n_dip, 1231 DDI_SERVICE_LOST); 1232 } 1233 nvme_free_cmd(cmd); 1234 return; 1235 } 1236 1237 1238 event.r = cmd->nc_cqe.cqe_dw0; 1239 1240 /* Clear CQE and re-submit the async request. */ 1241 bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 1242 ret = nvme_submit_cmd(nvme->n_adminq, cmd); 1243 1244 if (ret != DDI_SUCCESS) { 1245 dev_err(nvme->n_dip, CE_WARN, 1246 "!failed to resubmit async event request"); 1247 atomic_inc_32(&nvme->n_async_resubmit_failed); 1248 nvme_free_cmd(cmd); 1249 } 1250 1251 switch (event.b.ae_type) { 1252 case NVME_ASYNC_TYPE_ERROR: 1253 if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 1254 error_log = (nvme_error_log_entry_t *) 1255 nvme_get_logpage(nvme, event.b.ae_logpage); 1256 } else { 1257 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 1258 "async event reply: %d", event.b.ae_logpage); 1259 atomic_inc_32(&nvme->n_wrong_logpage); 1260 } 1261 1262 switch (event.b.ae_info) { 1263 case NVME_ASYNC_ERROR_INV_SQ: 1264 dev_err(nvme->n_dip, CE_PANIC, "programming error: " 1265 "invalid submission queue"); 1266 return; 1267 1268 case NVME_ASYNC_ERROR_INV_DBL: 1269 dev_err(nvme->n_dip, CE_PANIC, "programming error: " 1270 "invalid doorbell write value"); 1271 return; 1272 1273 case NVME_ASYNC_ERROR_DIAGFAIL: 1274 dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 1275 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1276 nvme->n_dead = B_TRUE; 1277 atomic_inc_32(&nvme->n_diagfail_event); 1278 break; 1279 1280 case NVME_ASYNC_ERROR_PERSISTENT: 1281 dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 1282 "device error"); 1283 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1284 nvme->n_dead = B_TRUE; 1285 atomic_inc_32(&nvme->n_persistent_event); 1286 break; 1287 1288 case NVME_ASYNC_ERROR_TRANSIENT: 1289 dev_err(nvme->n_dip, CE_WARN, "!transient internal " 1290 "device error"); 1291 /* TODO: send ereport */ 1292 atomic_inc_32(&nvme->n_transient_event); 1293 break; 1294 1295 case NVME_ASYNC_ERROR_FW_LOAD: 1296 dev_err(nvme->n_dip, CE_WARN, 1297 "!firmware image load error"); 1298 atomic_inc_32(&nvme->n_fw_load_event); 1299 break; 1300 } 1301 break; 1302 1303 case NVME_ASYNC_TYPE_HEALTH: 1304 if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 1305 health_log = (nvme_health_log_t *) 1306 nvme_get_logpage(nvme, event.b.ae_logpage, -1); 1307 } else { 1308 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 1309 "async event reply: %d", event.b.ae_logpage); 1310 atomic_inc_32(&nvme->n_wrong_logpage); 1311 } 1312 1313 switch (event.b.ae_info) { 1314 case NVME_ASYNC_HEALTH_RELIABILITY: 1315 dev_err(nvme->n_dip, CE_WARN, 1316 "!device reliability compromised"); 1317 /* TODO: send ereport */ 1318 atomic_inc_32(&nvme->n_reliability_event); 1319 break; 1320 1321 case NVME_ASYNC_HEALTH_TEMPERATURE: 1322 dev_err(nvme->n_dip, CE_WARN, 1323 "!temperature above threshold"); 1324 /* TODO: send ereport */ 1325 atomic_inc_32(&nvme->n_temperature_event); 1326 break; 1327 1328 case NVME_ASYNC_HEALTH_SPARE: 1329 dev_err(nvme->n_dip, CE_WARN, 1330 "!spare space below threshold"); 1331 /* TODO: send ereport */ 1332 atomic_inc_32(&nvme->n_spare_event); 1333 break; 1334 } 1335 break; 1336 1337 case NVME_ASYNC_TYPE_VENDOR: 1338 dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 1339 "received, info = %x, logpage = %x", event.b.ae_info, 1340 event.b.ae_logpage); 1341 atomic_inc_32(&nvme->n_vendor_event); 1342 break; 1343 1344 default: 1345 dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 1346 "type = %x, info = %x, logpage = %x", event.b.ae_type, 1347 event.b.ae_info, event.b.ae_logpage); 1348 atomic_inc_32(&nvme->n_unknown_event); 1349 break; 1350 } 1351 1352 if (error_log) 1353 kmem_free(error_log, sizeof (nvme_error_log_entry_t) * 1354 nvme->n_error_log_len); 1355 1356 if (health_log) 1357 kmem_free(health_log, sizeof (nvme_health_log_t)); 1358 } 1359 1360 static int 1361 nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 1362 { 1363 int ret; 1364 1365 mutex_enter(&cmd->nc_mutex); 1366 ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd); 1367 1368 if (ret != DDI_SUCCESS) { 1369 mutex_exit(&cmd->nc_mutex); 1370 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 1371 "!nvme_submit_cmd failed"); 1372 atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full); 1373 nvme_free_cmd(cmd); 1374 return (DDI_FAILURE); 1375 } 1376 1377 if (nvme_wait_cmd(cmd, sec) == B_FALSE) { 1378 /* 1379 * The command timed out. An abort command was posted that 1380 * will take care of the cleanup. 1381 */ 1382 return (DDI_FAILURE); 1383 } 1384 mutex_exit(&cmd->nc_mutex); 1385 1386 return (DDI_SUCCESS); 1387 } 1388 1389 static int 1390 nvme_async_event(nvme_t *nvme) 1391 { 1392 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1393 int ret; 1394 1395 cmd->nc_sqid = 0; 1396 cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 1397 cmd->nc_callback = nvme_async_event_task; 1398 1399 ret = nvme_submit_cmd(nvme->n_adminq, cmd); 1400 1401 if (ret != DDI_SUCCESS) { 1402 dev_err(nvme->n_dip, CE_WARN, 1403 "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT"); 1404 nvme_free_cmd(cmd); 1405 return (DDI_FAILURE); 1406 } 1407 1408 return (DDI_SUCCESS); 1409 } 1410 1411 static void * 1412 nvme_get_logpage(nvme_t *nvme, uint8_t logpage, ...) 1413 { 1414 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1415 void *buf = NULL; 1416 nvme_getlogpage_t getlogpage = { 0 }; 1417 size_t bufsize; 1418 va_list ap; 1419 1420 va_start(ap, logpage); 1421 1422 cmd->nc_sqid = 0; 1423 cmd->nc_callback = nvme_wakeup_cmd; 1424 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 1425 1426 getlogpage.b.lp_lid = logpage; 1427 1428 switch (logpage) { 1429 case NVME_LOGPAGE_ERROR: 1430 cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 1431 bufsize = nvme->n_error_log_len * 1432 sizeof (nvme_error_log_entry_t); 1433 break; 1434 1435 case NVME_LOGPAGE_HEALTH: 1436 cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 1437 bufsize = sizeof (nvme_health_log_t); 1438 break; 1439 1440 case NVME_LOGPAGE_FWSLOT: 1441 cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 1442 bufsize = sizeof (nvme_fwslot_log_t); 1443 break; 1444 1445 default: 1446 dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 1447 logpage); 1448 atomic_inc_32(&nvme->n_unknown_logpage); 1449 goto fail; 1450 } 1451 1452 va_end(ap); 1453 1454 getlogpage.b.lp_numd = bufsize / sizeof (uint32_t) - 1; 1455 1456 cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 1457 1458 if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t), 1459 DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 1460 dev_err(nvme->n_dip, CE_WARN, 1461 "!nvme_zalloc_dma failed for GET LOG PAGE"); 1462 goto fail; 1463 } 1464 1465 if (cmd->nc_dma->nd_ncookie > 2) { 1466 dev_err(nvme->n_dip, CE_WARN, 1467 "!too many DMA cookies for GET LOG PAGE"); 1468 atomic_inc_32(&nvme->n_too_many_cookies); 1469 goto fail; 1470 } 1471 1472 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 1473 if (cmd->nc_dma->nd_ncookie > 1) { 1474 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 1475 &cmd->nc_dma->nd_cookie); 1476 cmd->nc_sqe.sqe_dptr.d_prp[1] = 1477 cmd->nc_dma->nd_cookie.dmac_laddress; 1478 } 1479 1480 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1481 dev_err(nvme->n_dip, CE_WARN, 1482 "!nvme_admin_cmd failed for GET LOG PAGE"); 1483 return (NULL); 1484 } 1485 1486 if (nvme_check_cmd_status(cmd)) { 1487 dev_err(nvme->n_dip, CE_WARN, 1488 "!GET LOG PAGE failed with sct = %x, sc = %x", 1489 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1490 goto fail; 1491 } 1492 1493 buf = kmem_alloc(bufsize, KM_SLEEP); 1494 bcopy(cmd->nc_dma->nd_memp, buf, bufsize); 1495 1496 fail: 1497 nvme_free_cmd(cmd); 1498 1499 return (buf); 1500 } 1501 1502 static void * 1503 nvme_identify(nvme_t *nvme, uint32_t nsid) 1504 { 1505 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1506 void *buf = NULL; 1507 1508 cmd->nc_sqid = 0; 1509 cmd->nc_callback = nvme_wakeup_cmd; 1510 cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 1511 cmd->nc_sqe.sqe_nsid = nsid; 1512 cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 1513 1514 if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 1515 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 1516 dev_err(nvme->n_dip, CE_WARN, 1517 "!nvme_zalloc_dma failed for IDENTIFY"); 1518 goto fail; 1519 } 1520 1521 if (cmd->nc_dma->nd_ncookie > 2) { 1522 dev_err(nvme->n_dip, CE_WARN, 1523 "!too many DMA cookies for IDENTIFY"); 1524 atomic_inc_32(&nvme->n_too_many_cookies); 1525 goto fail; 1526 } 1527 1528 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 1529 if (cmd->nc_dma->nd_ncookie > 1) { 1530 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 1531 &cmd->nc_dma->nd_cookie); 1532 cmd->nc_sqe.sqe_dptr.d_prp[1] = 1533 cmd->nc_dma->nd_cookie.dmac_laddress; 1534 } 1535 1536 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1537 dev_err(nvme->n_dip, CE_WARN, 1538 "!nvme_admin_cmd failed for IDENTIFY"); 1539 return (NULL); 1540 } 1541 1542 if (nvme_check_cmd_status(cmd)) { 1543 dev_err(nvme->n_dip, CE_WARN, 1544 "!IDENTIFY failed with sct = %x, sc = %x", 1545 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1546 goto fail; 1547 } 1548 1549 buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 1550 bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE); 1551 1552 fail: 1553 nvme_free_cmd(cmd); 1554 1555 return (buf); 1556 } 1557 1558 static int 1559 nvme_set_nqueues(nvme_t *nvme, uint16_t nqueues) 1560 { 1561 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1562 nvme_nqueue_t nq = { 0 }; 1563 1564 nq.b.nq_nsq = nq.b.nq_ncq = nqueues - 1; 1565 1566 cmd->nc_sqid = 0; 1567 cmd->nc_callback = nvme_wakeup_cmd; 1568 cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 1569 cmd->nc_sqe.sqe_cdw10 = NVME_FEAT_NQUEUES; 1570 cmd->nc_sqe.sqe_cdw11 = nq.r; 1571 1572 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1573 dev_err(nvme->n_dip, CE_WARN, 1574 "!nvme_admin_cmd failed for SET FEATURES (NQUEUES)"); 1575 return (0); 1576 } 1577 1578 if (nvme_check_cmd_status(cmd)) { 1579 dev_err(nvme->n_dip, CE_WARN, 1580 "!SET FEATURES (NQUEUES) failed with sct = %x, sc = %x", 1581 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1582 nvme_free_cmd(cmd); 1583 return (0); 1584 } 1585 1586 nq.r = cmd->nc_cqe.cqe_dw0; 1587 nvme_free_cmd(cmd); 1588 1589 /* 1590 * Always use the same number of submission and completion queues, and 1591 * never use more than the requested number of queues. 1592 */ 1593 return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1)); 1594 } 1595 1596 static int 1597 nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 1598 { 1599 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1600 nvme_create_queue_dw10_t dw10 = { 0 }; 1601 nvme_create_cq_dw11_t c_dw11 = { 0 }; 1602 nvme_create_sq_dw11_t s_dw11 = { 0 }; 1603 1604 dw10.b.q_qid = idx; 1605 dw10.b.q_qsize = qp->nq_nentry - 1; 1606 1607 c_dw11.b.cq_pc = 1; 1608 c_dw11.b.cq_ien = 1; 1609 c_dw11.b.cq_iv = idx % nvme->n_intr_cnt; 1610 1611 cmd->nc_sqid = 0; 1612 cmd->nc_callback = nvme_wakeup_cmd; 1613 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 1614 cmd->nc_sqe.sqe_cdw10 = dw10.r; 1615 cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 1616 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; 1617 1618 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1619 dev_err(nvme->n_dip, CE_WARN, 1620 "!nvme_admin_cmd failed for CREATE CQUEUE"); 1621 return (DDI_FAILURE); 1622 } 1623 1624 if (nvme_check_cmd_status(cmd)) { 1625 dev_err(nvme->n_dip, CE_WARN, 1626 "!CREATE CQUEUE failed with sct = %x, sc = %x", 1627 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1628 nvme_free_cmd(cmd); 1629 return (DDI_FAILURE); 1630 } 1631 1632 nvme_free_cmd(cmd); 1633 1634 s_dw11.b.sq_pc = 1; 1635 s_dw11.b.sq_cqid = idx; 1636 1637 cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1638 cmd->nc_sqid = 0; 1639 cmd->nc_callback = nvme_wakeup_cmd; 1640 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 1641 cmd->nc_sqe.sqe_cdw10 = dw10.r; 1642 cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 1643 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 1644 1645 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1646 dev_err(nvme->n_dip, CE_WARN, 1647 "!nvme_admin_cmd failed for CREATE SQUEUE"); 1648 return (DDI_FAILURE); 1649 } 1650 1651 if (nvme_check_cmd_status(cmd)) { 1652 dev_err(nvme->n_dip, CE_WARN, 1653 "!CREATE SQUEUE failed with sct = %x, sc = %x", 1654 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1655 nvme_free_cmd(cmd); 1656 return (DDI_FAILURE); 1657 } 1658 1659 nvme_free_cmd(cmd); 1660 1661 return (DDI_SUCCESS); 1662 } 1663 1664 static boolean_t 1665 nvme_reset(nvme_t *nvme, boolean_t quiesce) 1666 { 1667 nvme_reg_csts_t csts; 1668 int i; 1669 1670 nvme_put32(nvme, NVME_REG_CC, 0); 1671 1672 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1673 if (csts.b.csts_rdy == 1) { 1674 nvme_put32(nvme, NVME_REG_CC, 0); 1675 for (i = 0; i != nvme->n_timeout * 10; i++) { 1676 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1677 if (csts.b.csts_rdy == 0) 1678 break; 1679 1680 if (quiesce) 1681 drv_usecwait(50000); 1682 else 1683 delay(drv_usectohz(50000)); 1684 } 1685 } 1686 1687 nvme_put32(nvme, NVME_REG_AQA, 0); 1688 nvme_put32(nvme, NVME_REG_ASQ, 0); 1689 nvme_put32(nvme, NVME_REG_ACQ, 0); 1690 1691 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1692 return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 1693 } 1694 1695 static void 1696 nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 1697 { 1698 nvme_reg_cc_t cc; 1699 nvme_reg_csts_t csts; 1700 int i; 1701 1702 ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 1703 1704 cc.r = nvme_get32(nvme, NVME_REG_CC); 1705 cc.b.cc_shn = mode & 0x3; 1706 nvme_put32(nvme, NVME_REG_CC, cc.r); 1707 1708 for (i = 0; i != 10; i++) { 1709 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1710 if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 1711 break; 1712 1713 if (quiesce) 1714 drv_usecwait(100000); 1715 else 1716 delay(drv_usectohz(100000)); 1717 } 1718 } 1719 1720 1721 static void 1722 nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 1723 { 1724 char model[sizeof (nvme->n_idctl->id_model) + 1]; 1725 char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 1726 1727 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 1728 bcopy(nvme->n_idctl->id_serial, serial, 1729 sizeof (nvme->n_idctl->id_serial)); 1730 1731 model[sizeof (nvme->n_idctl->id_model)] = '\0'; 1732 serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 1733 1734 (void) snprintf(nvme->n_ns[nsid - 1].ns_devid, 1735 sizeof (nvme->n_ns[0].ns_devid), "%4X-%s-%s-%X", 1736 nvme->n_idctl->id_vid, model, serial, nsid); 1737 } 1738 1739 static int 1740 nvme_init(nvme_t *nvme) 1741 { 1742 nvme_reg_cc_t cc = { 0 }; 1743 nvme_reg_aqa_t aqa = { 0 }; 1744 nvme_reg_asq_t asq = { 0 }; 1745 nvme_reg_acq_t acq = { 0 }; 1746 nvme_reg_cap_t cap; 1747 nvme_reg_vs_t vs; 1748 nvme_reg_csts_t csts; 1749 int i = 0; 1750 int nqueues; 1751 char model[sizeof (nvme->n_idctl->id_model) + 1]; 1752 char *vendor, *product; 1753 1754 /* Check controller version */ 1755 vs.r = nvme_get32(nvme, NVME_REG_VS); 1756 dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 1757 vs.b.vs_mjr, vs.b.vs_mnr); 1758 1759 if (nvme_version_major < vs.b.vs_mjr || 1760 (nvme_version_major == vs.b.vs_mjr && 1761 nvme_version_minor < vs.b.vs_mnr)) { 1762 dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d", 1763 nvme_version_major, nvme_version_minor); 1764 if (nvme->n_strict_version) 1765 goto fail; 1766 } 1767 1768 /* retrieve controller configuration */ 1769 cap.r = nvme_get64(nvme, NVME_REG_CAP); 1770 1771 if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 1772 dev_err(nvme->n_dip, CE_WARN, 1773 "!NVM command set not supported by hardware"); 1774 goto fail; 1775 } 1776 1777 nvme->n_nssr_supported = cap.b.cap_nssrs; 1778 nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 1779 nvme->n_timeout = cap.b.cap_to; 1780 nvme->n_arbitration_mechanisms = cap.b.cap_ams; 1781 nvme->n_cont_queues_reqd = cap.b.cap_cqr; 1782 nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 1783 1784 /* 1785 * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 1786 * the base page size of 4k (1<<12), so add 12 here to get the real 1787 * page size value. 1788 */ 1789 nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 1790 cap.b.cap_mpsmax + 12); 1791 nvme->n_pagesize = 1UL << (nvme->n_pageshift); 1792 1793 /* 1794 * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 1795 */ 1796 nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 1797 nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 1798 1799 /* 1800 * Set up PRP DMA to transfer 1 page-aligned page at a time. 1801 * Maxxfer may be increased after we identified the controller limits. 1802 */ 1803 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 1804 nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 1805 nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 1806 nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 1807 1808 /* 1809 * Reset controller if it's still in ready state. 1810 */ 1811 if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 1812 dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 1813 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1814 nvme->n_dead = B_TRUE; 1815 goto fail; 1816 } 1817 1818 /* 1819 * Create the admin queue pair. 1820 */ 1821 if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 1822 != DDI_SUCCESS) { 1823 dev_err(nvme->n_dip, CE_WARN, 1824 "!unable to allocate admin qpair"); 1825 goto fail; 1826 } 1827 nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 1828 nvme->n_ioq[0] = nvme->n_adminq; 1829 1830 nvme->n_progress |= NVME_ADMIN_QUEUE; 1831 1832 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 1833 "admin-queue-len", nvme->n_admin_queue_len); 1834 1835 aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 1836 asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 1837 acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress; 1838 1839 ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 1840 ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 1841 1842 nvme_put32(nvme, NVME_REG_AQA, aqa.r); 1843 nvme_put64(nvme, NVME_REG_ASQ, asq); 1844 nvme_put64(nvme, NVME_REG_ACQ, acq); 1845 1846 cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 1847 cc.b.cc_css = 0; /* use NVM command set */ 1848 cc.b.cc_mps = nvme->n_pageshift - 12; 1849 cc.b.cc_shn = 0; /* no shutdown in progress */ 1850 cc.b.cc_en = 1; /* enable controller */ 1851 cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 1852 cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 1853 1854 nvme_put32(nvme, NVME_REG_CC, cc.r); 1855 1856 /* 1857 * Wait for the controller to become ready. 1858 */ 1859 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1860 if (csts.b.csts_rdy == 0) { 1861 for (i = 0; i != nvme->n_timeout * 10; i++) { 1862 delay(drv_usectohz(50000)); 1863 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1864 1865 if (csts.b.csts_cfs == 1) { 1866 dev_err(nvme->n_dip, CE_WARN, 1867 "!controller fatal status at init"); 1868 ddi_fm_service_impact(nvme->n_dip, 1869 DDI_SERVICE_LOST); 1870 nvme->n_dead = B_TRUE; 1871 goto fail; 1872 } 1873 1874 if (csts.b.csts_rdy == 1) 1875 break; 1876 } 1877 } 1878 1879 if (csts.b.csts_rdy == 0) { 1880 dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 1881 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1882 nvme->n_dead = B_TRUE; 1883 goto fail; 1884 } 1885 1886 /* 1887 * Assume an abort command limit of 1. We'll destroy and re-init 1888 * that later when we know the true abort command limit. 1889 */ 1890 sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 1891 1892 /* 1893 * Setup initial interrupt for admin queue. 1894 */ 1895 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 1896 != DDI_SUCCESS) && 1897 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 1898 != DDI_SUCCESS) && 1899 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 1900 != DDI_SUCCESS)) { 1901 dev_err(nvme->n_dip, CE_WARN, 1902 "!failed to setup initial interrupt"); 1903 goto fail; 1904 } 1905 1906 /* 1907 * Post an asynchronous event command to catch errors. 1908 */ 1909 if (nvme_async_event(nvme) != DDI_SUCCESS) { 1910 dev_err(nvme->n_dip, CE_WARN, 1911 "!failed to post async event"); 1912 goto fail; 1913 } 1914 1915 /* 1916 * Identify Controller 1917 */ 1918 nvme->n_idctl = nvme_identify(nvme, 0); 1919 if (nvme->n_idctl == NULL) { 1920 dev_err(nvme->n_dip, CE_WARN, 1921 "!failed to identify controller"); 1922 goto fail; 1923 } 1924 1925 /* 1926 * Get Vendor & Product ID 1927 */ 1928 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 1929 model[sizeof (nvme->n_idctl->id_model)] = '\0'; 1930 sata_split_model(model, &vendor, &product); 1931 1932 if (vendor == NULL) 1933 nvme->n_vendor = strdup("NVMe"); 1934 else 1935 nvme->n_vendor = strdup(vendor); 1936 1937 nvme->n_product = strdup(product); 1938 1939 /* 1940 * Get controller limits. 1941 */ 1942 nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 1943 MIN(nvme->n_admin_queue_len / 10, 1944 MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 1945 1946 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 1947 "async-event-limit", nvme->n_async_event_limit); 1948 1949 nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 1950 1951 /* 1952 * Reinitialize the semaphore with the true abort command limit 1953 * supported by the hardware. It's not necessary to disable interrupts 1954 * as only command aborts use the semaphore, and no commands are 1955 * executed or aborted while we're here. 1956 */ 1957 sema_destroy(&nvme->n_abort_sema); 1958 sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 1959 SEMA_DRIVER, NULL); 1960 1961 nvme->n_progress |= NVME_CTRL_LIMITS; 1962 1963 if (nvme->n_idctl->id_mdts == 0) 1964 nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 1965 else 1966 nvme->n_max_data_transfer_size = 1967 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 1968 1969 nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 1970 1971 /* 1972 * Limit n_max_data_transfer_size to what we can handle in one PRP. 1973 * Chained PRPs are currently unsupported. 1974 * 1975 * This is a no-op on hardware which doesn't support a transfer size 1976 * big enough to require chained PRPs. 1977 */ 1978 nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 1979 (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 1980 1981 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 1982 1983 /* 1984 * Make sure the minimum/maximum queue entry sizes are not 1985 * larger/smaller than the default. 1986 */ 1987 1988 if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 1989 ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 1990 ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 1991 ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 1992 goto fail; 1993 1994 /* 1995 * Check for the presence of a Volatile Write Cache. If present, 1996 * enable it by default. 1997 */ 1998 if (nvme->n_idctl->id_vwc.vwc_present == 0) { 1999 nvme->n_volatile_write_cache_enabled = B_FALSE; 2000 nvme_bd_ops.o_sync_cache = NULL; 2001 } else { 2002 /* 2003 * TODO: send SET FEATURES to enable VWC 2004 * (have no hardware to test this) 2005 */ 2006 nvme->n_volatile_write_cache_enabled = B_FALSE; 2007 nvme_bd_ops.o_sync_cache = NULL; 2008 } 2009 2010 /* 2011 * Grab a copy of all mandatory log pages. 2012 * 2013 * TODO: should go away once user space tool exists to print logs 2014 */ 2015 nvme->n_error_log = (nvme_error_log_entry_t *) 2016 nvme_get_logpage(nvme, NVME_LOGPAGE_ERROR); 2017 nvme->n_health_log = (nvme_health_log_t *) 2018 nvme_get_logpage(nvme, NVME_LOGPAGE_HEALTH, -1); 2019 nvme->n_fwslot_log = (nvme_fwslot_log_t *) 2020 nvme_get_logpage(nvme, NVME_LOGPAGE_FWSLOT); 2021 2022 /* 2023 * Identify Namespaces 2024 */ 2025 nvme->n_namespace_count = nvme->n_idctl->id_nn; 2026 nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 2027 nvme->n_namespace_count, KM_SLEEP); 2028 2029 for (i = 0; i != nvme->n_namespace_count; i++) { 2030 nvme_identify_nsid_t *idns; 2031 int last_rp; 2032 2033 nvme->n_ns[i].ns_nvme = nvme; 2034 nvme->n_ns[i].ns_idns = idns = nvme_identify(nvme, i + 1); 2035 2036 if (idns == NULL) { 2037 dev_err(nvme->n_dip, CE_WARN, 2038 "!failed to identify namespace %d", i + 1); 2039 goto fail; 2040 } 2041 2042 nvme->n_ns[i].ns_id = i + 1; 2043 nvme->n_ns[i].ns_block_count = idns->id_nsize; 2044 nvme->n_ns[i].ns_block_size = 2045 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 2046 nvme->n_ns[i].ns_best_block_size = nvme->n_ns[i].ns_block_size; 2047 2048 nvme_prepare_devid(nvme, nvme->n_ns[i].ns_id); 2049 2050 /* 2051 * Find the LBA format with no metadata and the best relative 2052 * performance. A value of 3 means "degraded", 0 is best. 2053 */ 2054 last_rp = 3; 2055 for (int j = 0; j <= idns->id_nlbaf; j++) { 2056 if (idns->id_lbaf[j].lbaf_lbads == 0) 2057 break; 2058 if (idns->id_lbaf[j].lbaf_ms != 0) 2059 continue; 2060 if (idns->id_lbaf[j].lbaf_rp >= last_rp) 2061 continue; 2062 last_rp = idns->id_lbaf[j].lbaf_rp; 2063 nvme->n_ns[i].ns_best_block_size = 2064 1 << idns->id_lbaf[j].lbaf_lbads; 2065 } 2066 2067 /* 2068 * We currently don't support namespaces that use either: 2069 * - thin provisioning 2070 * - extended LBAs 2071 * - protection information 2072 */ 2073 if (idns->id_nsfeat.f_thin || 2074 idns->id_flbas.lba_extlba || 2075 idns->id_dps.dp_pinfo) { 2076 dev_err(nvme->n_dip, CE_WARN, 2077 "!ignoring namespace %d, unsupported features: " 2078 "thin = %d, extlba = %d, pinfo = %d", i + 1, 2079 idns->id_nsfeat.f_thin, idns->id_flbas.lba_extlba, 2080 idns->id_dps.dp_pinfo); 2081 nvme->n_ns[i].ns_ignore = B_TRUE; 2082 } 2083 } 2084 2085 /* 2086 * Try to set up MSI/MSI-X interrupts. 2087 */ 2088 if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 2089 != 0) { 2090 nvme_release_interrupts(nvme); 2091 2092 nqueues = MIN(UINT16_MAX, ncpus); 2093 2094 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 2095 nqueues) != DDI_SUCCESS) && 2096 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 2097 nqueues) != DDI_SUCCESS)) { 2098 dev_err(nvme->n_dip, CE_WARN, 2099 "!failed to setup MSI/MSI-X interrupts"); 2100 goto fail; 2101 } 2102 } 2103 2104 nqueues = nvme->n_intr_cnt; 2105 2106 /* 2107 * Create I/O queue pairs. 2108 */ 2109 nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues); 2110 if (nvme->n_ioq_count == 0) { 2111 dev_err(nvme->n_dip, CE_WARN, 2112 "!failed to set number of I/O queues to %d", nqueues); 2113 goto fail; 2114 } 2115 2116 /* 2117 * Reallocate I/O queue array 2118 */ 2119 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 2120 nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 2121 (nvme->n_ioq_count + 1), KM_SLEEP); 2122 nvme->n_ioq[0] = nvme->n_adminq; 2123 2124 /* 2125 * If we got less queues than we asked for we might as well give 2126 * some of the interrupt vectors back to the system. 2127 */ 2128 if (nvme->n_ioq_count < nqueues) { 2129 nvme_release_interrupts(nvme); 2130 2131 if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 2132 nvme->n_ioq_count) != DDI_SUCCESS) { 2133 dev_err(nvme->n_dip, CE_WARN, 2134 "!failed to reduce number of interrupts"); 2135 goto fail; 2136 } 2137 } 2138 2139 /* 2140 * Alloc & register I/O queue pairs 2141 */ 2142 nvme->n_io_queue_len = 2143 MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries); 2144 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len", 2145 nvme->n_io_queue_len); 2146 2147 for (i = 1; i != nvme->n_ioq_count + 1; i++) { 2148 if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len, 2149 &nvme->n_ioq[i], i) != DDI_SUCCESS) { 2150 dev_err(nvme->n_dip, CE_WARN, 2151 "!unable to allocate I/O qpair %d", i); 2152 goto fail; 2153 } 2154 2155 if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) 2156 != DDI_SUCCESS) { 2157 dev_err(nvme->n_dip, CE_WARN, 2158 "!unable to create I/O qpair %d", i); 2159 goto fail; 2160 } 2161 } 2162 2163 /* 2164 * Post more asynchronous events commands to reduce event reporting 2165 * latency as suggested by the spec. 2166 */ 2167 for (i = 1; i != nvme->n_async_event_limit; i++) { 2168 if (nvme_async_event(nvme) != DDI_SUCCESS) { 2169 dev_err(nvme->n_dip, CE_WARN, 2170 "!failed to post async event %d", i); 2171 goto fail; 2172 } 2173 } 2174 2175 return (DDI_SUCCESS); 2176 2177 fail: 2178 (void) nvme_reset(nvme, B_FALSE); 2179 return (DDI_FAILURE); 2180 } 2181 2182 static uint_t 2183 nvme_intr(caddr_t arg1, caddr_t arg2) 2184 { 2185 /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 2186 nvme_t *nvme = (nvme_t *)arg1; 2187 int inum = (int)(uintptr_t)arg2; 2188 int ccnt = 0; 2189 int qnum; 2190 nvme_cmd_t *cmd; 2191 2192 if (inum >= nvme->n_intr_cnt) 2193 return (DDI_INTR_UNCLAIMED); 2194 2195 /* 2196 * The interrupt vector a queue uses is calculated as queue_idx % 2197 * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 2198 * in steps of n_intr_cnt to process all queues using this vector. 2199 */ 2200 for (qnum = inum; 2201 qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL; 2202 qnum += nvme->n_intr_cnt) { 2203 while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { 2204 taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 2205 cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 2206 ccnt++; 2207 } 2208 } 2209 2210 return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 2211 } 2212 2213 static void 2214 nvme_release_interrupts(nvme_t *nvme) 2215 { 2216 int i; 2217 2218 for (i = 0; i < nvme->n_intr_cnt; i++) { 2219 if (nvme->n_inth[i] == NULL) 2220 break; 2221 2222 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 2223 (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 2224 else 2225 (void) ddi_intr_disable(nvme->n_inth[i]); 2226 2227 (void) ddi_intr_remove_handler(nvme->n_inth[i]); 2228 (void) ddi_intr_free(nvme->n_inth[i]); 2229 } 2230 2231 kmem_free(nvme->n_inth, nvme->n_inth_sz); 2232 nvme->n_inth = NULL; 2233 nvme->n_inth_sz = 0; 2234 2235 nvme->n_progress &= ~NVME_INTERRUPTS; 2236 } 2237 2238 static int 2239 nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 2240 { 2241 int nintrs, navail, count; 2242 int ret; 2243 int i; 2244 2245 if (nvme->n_intr_types == 0) { 2246 ret = ddi_intr_get_supported_types(nvme->n_dip, 2247 &nvme->n_intr_types); 2248 if (ret != DDI_SUCCESS) { 2249 dev_err(nvme->n_dip, CE_WARN, 2250 "!%s: ddi_intr_get_supported types failed", 2251 __func__); 2252 return (ret); 2253 } 2254 } 2255 2256 if ((nvme->n_intr_types & intr_type) == 0) 2257 return (DDI_FAILURE); 2258 2259 ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 2260 if (ret != DDI_SUCCESS) { 2261 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 2262 __func__); 2263 return (ret); 2264 } 2265 2266 ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 2267 if (ret != DDI_SUCCESS) { 2268 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 2269 __func__); 2270 return (ret); 2271 } 2272 2273 /* We want at most one interrupt per queue pair. */ 2274 if (navail > nqpairs) 2275 navail = nqpairs; 2276 2277 nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 2278 nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 2279 2280 ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 2281 &count, 0); 2282 if (ret != DDI_SUCCESS) { 2283 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 2284 __func__); 2285 goto fail; 2286 } 2287 2288 nvme->n_intr_cnt = count; 2289 2290 ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 2291 if (ret != DDI_SUCCESS) { 2292 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 2293 __func__); 2294 goto fail; 2295 } 2296 2297 for (i = 0; i < count; i++) { 2298 ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 2299 (void *)nvme, (void *)(uintptr_t)i); 2300 if (ret != DDI_SUCCESS) { 2301 dev_err(nvme->n_dip, CE_WARN, 2302 "!%s: ddi_intr_add_handler failed", __func__); 2303 goto fail; 2304 } 2305 } 2306 2307 (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 2308 2309 for (i = 0; i < count; i++) { 2310 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 2311 ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 2312 else 2313 ret = ddi_intr_enable(nvme->n_inth[i]); 2314 2315 if (ret != DDI_SUCCESS) { 2316 dev_err(nvme->n_dip, CE_WARN, 2317 "!%s: enabling interrupt %d failed", __func__, i); 2318 goto fail; 2319 } 2320 } 2321 2322 nvme->n_intr_type = intr_type; 2323 2324 nvme->n_progress |= NVME_INTERRUPTS; 2325 2326 return (DDI_SUCCESS); 2327 2328 fail: 2329 nvme_release_interrupts(nvme); 2330 2331 return (ret); 2332 } 2333 2334 static int 2335 nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 2336 { 2337 _NOTE(ARGUNUSED(arg)); 2338 2339 pci_ereport_post(dip, fm_error, NULL); 2340 return (fm_error->fme_status); 2341 } 2342 2343 static int 2344 nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 2345 { 2346 nvme_t *nvme; 2347 int instance; 2348 int nregs; 2349 off_t regsize; 2350 int i; 2351 char name[32]; 2352 2353 if (cmd != DDI_ATTACH) 2354 return (DDI_FAILURE); 2355 2356 instance = ddi_get_instance(dip); 2357 2358 if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 2359 return (DDI_FAILURE); 2360 2361 nvme = ddi_get_soft_state(nvme_state, instance); 2362 ddi_set_driver_private(dip, nvme); 2363 nvme->n_dip = dip; 2364 2365 nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2366 DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 2367 nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 2368 dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 2369 B_TRUE : B_FALSE; 2370 nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2371 DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 2372 nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2373 DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN); 2374 nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2375 DDI_PROP_DONTPASS, "async-event-limit", 2376 NVME_DEFAULT_ASYNC_EVENT_LIMIT); 2377 2378 if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 2379 nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 2380 else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 2381 nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 2382 2383 if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN) 2384 nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN; 2385 2386 if (nvme->n_async_event_limit < 1) 2387 nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 2388 2389 nvme->n_reg_acc_attr = nvme_reg_acc_attr; 2390 nvme->n_queue_dma_attr = nvme_queue_dma_attr; 2391 nvme->n_prp_dma_attr = nvme_prp_dma_attr; 2392 nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 2393 2394 /* 2395 * Setup FMA support. 2396 */ 2397 nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 2398 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 2399 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 2400 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 2401 2402 ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 2403 2404 if (nvme->n_fm_cap) { 2405 if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 2406 nvme->n_reg_acc_attr.devacc_attr_access = 2407 DDI_FLAGERR_ACC; 2408 2409 if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 2410 nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 2411 nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 2412 } 2413 2414 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 2415 DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 2416 pci_ereport_setup(dip); 2417 2418 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 2419 ddi_fm_handler_register(dip, nvme_fm_errcb, 2420 (void *)nvme); 2421 } 2422 2423 nvme->n_progress |= NVME_FMA_INIT; 2424 2425 /* 2426 * The spec defines several register sets. Only the controller 2427 * registers (set 1) are currently used. 2428 */ 2429 if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 2430 nregs < 2 || 2431 ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 2432 goto fail; 2433 2434 if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 2435 &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 2436 dev_err(dip, CE_WARN, "!failed to map regset 1"); 2437 goto fail; 2438 } 2439 2440 nvme->n_progress |= NVME_REGS_MAPPED; 2441 2442 /* 2443 * Create taskq for command completion. 2444 */ 2445 (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 2446 ddi_driver_name(dip), ddi_get_instance(dip)); 2447 nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 2448 TASKQ_DEFAULTPRI, 0); 2449 if (nvme->n_cmd_taskq == NULL) { 2450 dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 2451 goto fail; 2452 } 2453 2454 2455 if (nvme_init(nvme) != DDI_SUCCESS) 2456 goto fail; 2457 2458 /* 2459 * Attach the blkdev driver for each namespace. 2460 */ 2461 for (i = 0; i != nvme->n_namespace_count; i++) { 2462 if (nvme->n_ns[i].ns_ignore) 2463 continue; 2464 2465 nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 2466 &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 2467 2468 if (nvme->n_ns[i].ns_bd_hdl == NULL) { 2469 dev_err(dip, CE_WARN, 2470 "!failed to get blkdev handle for namespace %d", i); 2471 goto fail; 2472 } 2473 2474 if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 2475 != DDI_SUCCESS) { 2476 dev_err(dip, CE_WARN, 2477 "!failed to attach blkdev handle for namespace %d", 2478 i); 2479 goto fail; 2480 } 2481 } 2482 2483 return (DDI_SUCCESS); 2484 2485 fail: 2486 /* attach successful anyway so that FMA can retire the device */ 2487 if (nvme->n_dead) 2488 return (DDI_SUCCESS); 2489 2490 (void) nvme_detach(dip, DDI_DETACH); 2491 2492 return (DDI_FAILURE); 2493 } 2494 2495 static int 2496 nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 2497 { 2498 int instance, i; 2499 nvme_t *nvme; 2500 2501 if (cmd != DDI_DETACH) 2502 return (DDI_FAILURE); 2503 2504 instance = ddi_get_instance(dip); 2505 2506 nvme = ddi_get_soft_state(nvme_state, instance); 2507 2508 if (nvme == NULL) 2509 return (DDI_FAILURE); 2510 2511 if (nvme->n_ns) { 2512 for (i = 0; i != nvme->n_namespace_count; i++) { 2513 if (nvme->n_ns[i].ns_bd_hdl) { 2514 (void) bd_detach_handle( 2515 nvme->n_ns[i].ns_bd_hdl); 2516 bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 2517 } 2518 2519 if (nvme->n_ns[i].ns_idns) 2520 kmem_free(nvme->n_ns[i].ns_idns, 2521 sizeof (nvme_identify_nsid_t)); 2522 } 2523 2524 kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 2525 nvme->n_namespace_count); 2526 } 2527 2528 if (nvme->n_progress & NVME_INTERRUPTS) 2529 nvme_release_interrupts(nvme); 2530 2531 if (nvme->n_cmd_taskq) 2532 ddi_taskq_wait(nvme->n_cmd_taskq); 2533 2534 if (nvme->n_ioq_count > 0) { 2535 for (i = 1; i != nvme->n_ioq_count + 1; i++) { 2536 if (nvme->n_ioq[i] != NULL) { 2537 /* TODO: send destroy queue commands */ 2538 nvme_free_qpair(nvme->n_ioq[i]); 2539 } 2540 } 2541 2542 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 2543 (nvme->n_ioq_count + 1)); 2544 } 2545 2546 if (nvme->n_progress & NVME_REGS_MAPPED) { 2547 nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 2548 (void) nvme_reset(nvme, B_FALSE); 2549 } 2550 2551 if (nvme->n_cmd_taskq) 2552 ddi_taskq_destroy(nvme->n_cmd_taskq); 2553 2554 if (nvme->n_progress & NVME_CTRL_LIMITS) 2555 sema_destroy(&nvme->n_abort_sema); 2556 2557 if (nvme->n_progress & NVME_ADMIN_QUEUE) 2558 nvme_free_qpair(nvme->n_adminq); 2559 2560 if (nvme->n_idctl) 2561 kmem_free(nvme->n_idctl, sizeof (nvme_identify_ctrl_t)); 2562 2563 if (nvme->n_progress & NVME_REGS_MAPPED) 2564 ddi_regs_map_free(&nvme->n_regh); 2565 2566 if (nvme->n_progress & NVME_FMA_INIT) { 2567 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 2568 ddi_fm_handler_unregister(nvme->n_dip); 2569 2570 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 2571 DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 2572 pci_ereport_teardown(nvme->n_dip); 2573 2574 ddi_fm_fini(nvme->n_dip); 2575 } 2576 2577 if (nvme->n_vendor != NULL) 2578 strfree(nvme->n_vendor); 2579 2580 if (nvme->n_product != NULL) 2581 strfree(nvme->n_product); 2582 2583 ddi_soft_state_free(nvme_state, instance); 2584 2585 return (DDI_SUCCESS); 2586 } 2587 2588 static int 2589 nvme_quiesce(dev_info_t *dip) 2590 { 2591 int instance; 2592 nvme_t *nvme; 2593 2594 instance = ddi_get_instance(dip); 2595 2596 nvme = ddi_get_soft_state(nvme_state, instance); 2597 2598 if (nvme == NULL) 2599 return (DDI_FAILURE); 2600 2601 nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 2602 2603 (void) nvme_reset(nvme, B_TRUE); 2604 2605 return (DDI_FAILURE); 2606 } 2607 2608 static int 2609 nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 2610 { 2611 nvme_t *nvme = cmd->nc_nvme; 2612 int nprp_page, nprp; 2613 uint64_t *prp; 2614 2615 if (xfer->x_ndmac == 0) 2616 return (DDI_FAILURE); 2617 2618 cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 2619 ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 2620 2621 if (xfer->x_ndmac == 1) { 2622 cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 2623 return (DDI_SUCCESS); 2624 } else if (xfer->x_ndmac == 2) { 2625 cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 2626 return (DDI_SUCCESS); 2627 } 2628 2629 xfer->x_ndmac--; 2630 2631 nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1; 2632 ASSERT(nprp_page > 0); 2633 nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 2634 2635 /* 2636 * We currently don't support chained PRPs and set up our DMA 2637 * attributes to reflect that. If we still get an I/O request 2638 * that needs a chained PRP something is very wrong. 2639 */ 2640 VERIFY(nprp == 1); 2641 2642 if (nvme_zalloc_dma(nvme, nvme->n_pagesize * nprp, DDI_DMA_READ, 2643 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 2644 dev_err(nvme->n_dip, CE_WARN, "!%s: nvme_zalloc_dma failed", 2645 __func__); 2646 return (DDI_FAILURE); 2647 } 2648 2649 cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 2650 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, &cmd->nc_dma->nd_cookie); 2651 2652 /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 2653 for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 2654 xfer->x_ndmac > 0; 2655 prp++, xfer->x_ndmac--) { 2656 *prp = xfer->x_dmac.dmac_laddress; 2657 ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 2658 } 2659 2660 (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 2661 DDI_DMA_SYNC_FORDEV); 2662 return (DDI_SUCCESS); 2663 } 2664 2665 static nvme_cmd_t * 2666 nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 2667 { 2668 nvme_t *nvme = ns->ns_nvme; 2669 nvme_cmd_t *cmd; 2670 2671 /* 2672 * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 2673 */ 2674 cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 2675 KM_NOSLEEP : KM_SLEEP); 2676 2677 if (cmd == NULL) 2678 return (NULL); 2679 2680 cmd->nc_sqe.sqe_opc = opc; 2681 cmd->nc_callback = nvme_bd_xfer_done; 2682 cmd->nc_xfer = xfer; 2683 2684 switch (opc) { 2685 case NVME_OPC_NVM_WRITE: 2686 case NVME_OPC_NVM_READ: 2687 VERIFY(xfer->x_nblks <= 0x10000); 2688 2689 cmd->nc_sqe.sqe_nsid = ns->ns_id; 2690 2691 cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 2692 cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 2693 cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 2694 2695 if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 2696 goto fail; 2697 break; 2698 2699 case NVME_OPC_NVM_FLUSH: 2700 cmd->nc_sqe.sqe_nsid = ns->ns_id; 2701 break; 2702 2703 default: 2704 goto fail; 2705 } 2706 2707 return (cmd); 2708 2709 fail: 2710 nvme_free_cmd(cmd); 2711 return (NULL); 2712 } 2713 2714 static void 2715 nvme_bd_xfer_done(void *arg) 2716 { 2717 nvme_cmd_t *cmd = arg; 2718 bd_xfer_t *xfer = cmd->nc_xfer; 2719 int error = 0; 2720 2721 error = nvme_check_cmd_status(cmd); 2722 nvme_free_cmd(cmd); 2723 2724 bd_xfer_done(xfer, error); 2725 } 2726 2727 static void 2728 nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 2729 { 2730 nvme_namespace_t *ns = arg; 2731 nvme_t *nvme = ns->ns_nvme; 2732 2733 /* 2734 * blkdev maintains one queue size per instance (namespace), 2735 * but all namespace share the I/O queues. 2736 * TODO: need to figure out a sane default, or use per-NS I/O queues, 2737 * or change blkdev to handle EAGAIN 2738 */ 2739 drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len 2740 / nvme->n_namespace_count; 2741 2742 /* 2743 * d_maxxfer is not set, which means the value is taken from the DMA 2744 * attributes specified to bd_alloc_handle. 2745 */ 2746 2747 drive->d_removable = B_FALSE; 2748 drive->d_hotpluggable = B_FALSE; 2749 2750 drive->d_target = ns->ns_id; 2751 drive->d_lun = 0; 2752 2753 drive->d_model = nvme->n_idctl->id_model; 2754 drive->d_model_len = sizeof (nvme->n_idctl->id_model); 2755 drive->d_vendor = nvme->n_vendor; 2756 drive->d_vendor_len = strlen(nvme->n_vendor); 2757 drive->d_product = nvme->n_product; 2758 drive->d_product_len = strlen(nvme->n_product); 2759 drive->d_serial = nvme->n_idctl->id_serial; 2760 drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 2761 drive->d_revision = nvme->n_idctl->id_fwrev; 2762 drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 2763 } 2764 2765 static int 2766 nvme_bd_mediainfo(void *arg, bd_media_t *media) 2767 { 2768 nvme_namespace_t *ns = arg; 2769 2770 media->m_nblks = ns->ns_block_count; 2771 media->m_blksize = ns->ns_block_size; 2772 media->m_readonly = B_FALSE; 2773 media->m_solidstate = B_TRUE; 2774 2775 media->m_pblksize = ns->ns_best_block_size; 2776 2777 return (0); 2778 } 2779 2780 static int 2781 nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 2782 { 2783 nvme_t *nvme = ns->ns_nvme; 2784 nvme_cmd_t *cmd; 2785 2786 if (nvme->n_dead) 2787 return (EIO); 2788 2789 /* No polling for now */ 2790 if (xfer->x_flags & BD_XFER_POLL) 2791 return (EIO); 2792 2793 cmd = nvme_create_nvm_cmd(ns, opc, xfer); 2794 if (cmd == NULL) 2795 return (ENOMEM); 2796 2797 cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 2798 ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 2799 2800 if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd) 2801 != DDI_SUCCESS) 2802 return (EAGAIN); 2803 2804 return (0); 2805 } 2806 2807 static int 2808 nvme_bd_read(void *arg, bd_xfer_t *xfer) 2809 { 2810 nvme_namespace_t *ns = arg; 2811 2812 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 2813 } 2814 2815 static int 2816 nvme_bd_write(void *arg, bd_xfer_t *xfer) 2817 { 2818 nvme_namespace_t *ns = arg; 2819 2820 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 2821 } 2822 2823 static int 2824 nvme_bd_sync(void *arg, bd_xfer_t *xfer) 2825 { 2826 nvme_namespace_t *ns = arg; 2827 2828 if (ns->ns_nvme->n_dead) 2829 return (EIO); 2830 2831 /* 2832 * If the volatile write cache isn't enabled the FLUSH command is a 2833 * no-op, so we can take a shortcut here. 2834 */ 2835 if (ns->ns_nvme->n_volatile_write_cache_enabled == B_FALSE) { 2836 bd_xfer_done(xfer, ENOTSUP); 2837 return (0); 2838 } 2839 2840 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 2841 } 2842 2843 static int 2844 nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 2845 { 2846 nvme_namespace_t *ns = arg; 2847 2848 return (ddi_devid_init(devinfo, DEVID_ENCAP, strlen(ns->ns_devid), 2849 ns->ns_devid, devid)); 2850 } 2851