Lines Matching refs:nvme
563 #error nvme driver needs porting for big-endian platforms
825 #define NVME_BUMP_STAT(nvme, stat) \ argument
826 atomic_inc_64(&nvme->n_device_stat.nds_ ## stat.value.ui64)
1252 nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) in nvme_put64() argument
1254 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); in nvme_put64()
1257 ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); in nvme_put64()
1261 nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) in nvme_put32() argument
1263 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); in nvme_put32()
1266 ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); in nvme_put32()
1270 nvme_get64(nvme_t *nvme, uintptr_t reg) in nvme_get64() argument
1274 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); in nvme_get64()
1277 val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); in nvme_get64()
1283 nvme_get32(nvme_t *nvme, uintptr_t reg) in nvme_get32() argument
1287 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); in nvme_get32()
1290 val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); in nvme_get32()
1312 nvme_mgmt_unlock(nvme_t *nvme) in nvme_mgmt_unlock() argument
1314 nvme_mgmt_lock_t *lock = &nvme->n_mgmt; in nvme_mgmt_unlock()
1321 nvme_mgmt_lock_held(const nvme_t *nvme) in nvme_mgmt_lock_held() argument
1323 return (MUTEX_HELD(&nvme->n_mgmt.nml_lock) != 0); in nvme_mgmt_lock_held()
1327 nvme_mgmt_lock(nvme_t *nvme, nvme_mgmt_lock_level_t level) in nvme_mgmt_lock() argument
1329 nvme_mgmt_lock_t *lock = &nvme->n_mgmt; in nvme_mgmt_lock()
1346 nvme_mgmt_bd_start(nvme_t *nvme) in nvme_mgmt_bd_start() argument
1348 nvme_mgmt_lock_t *lock = &nvme->n_mgmt; in nvme_mgmt_bd_start()
1357 nvme_mgmt_bd_end(nvme_t *nvme) in nvme_mgmt_bd_end() argument
1359 nvme_mgmt_lock_t *lock = &nvme->n_mgmt; in nvme_mgmt_bd_end()
1388 nvme_ctrl_mark_dead(nvme_t *nvme, boolean_t removed) in nvme_ctrl_mark_dead() argument
1396 was_dead = atomic_cas_32((volatile uint32_t *)&nvme->n_dead, B_FALSE, in nvme_ctrl_mark_dead()
1406 nvme->n_dead_status = NVME_IOCTL_E_CTRL_GONE; in nvme_ctrl_mark_dead()
1419 ASSERT3U(nvme->n_dead_status, ==, NVME_IOCTL_E_CTRL_DEAD); in nvme_ctrl_mark_dead()
1420 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); in nvme_ctrl_mark_dead()
1423 taskq_dispatch_ent(nvme_dead_taskq, nvme_rwlock_ctrl_dead, nvme, in nvme_ctrl_mark_dead()
1424 TQ_NOSLEEP, &nvme->n_dead_tqent); in nvme_ctrl_mark_dead()
1428 nvme_ctrl_is_gone(const nvme_t *nvme) in nvme_ctrl_is_gone() argument
1430 if (nvme->n_dead && nvme->n_dead_status == NVME_IOCTL_E_CTRL_GONE) in nvme_ctrl_is_gone()
1437 nvme_check_regs_hdl(nvme_t *nvme) in nvme_check_regs_hdl() argument
1441 ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); in nvme_check_regs_hdl()
1492 nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, in nvme_alloc_dma_common() argument
1495 if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, in nvme_alloc_dma_common()
1502 dev_err(nvme->n_dip, CE_PANIC, in nvme_alloc_dma_common()
1511 (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, in nvme_alloc_dma_common()
1518 dev_err(nvme->n_dip, CE_WARN, in nvme_alloc_dma_common()
1520 NVME_BUMP_STAT(nvme, dma_bind_err); in nvme_alloc_dma_common()
1529 nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, in nvme_zalloc_dma() argument
1534 if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != in nvme_zalloc_dma()
1551 nvme_t *nvme = (nvme_t *)private; in nvme_prp_dma_constructor() local
1556 if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, in nvme_prp_dma_constructor()
1557 DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { in nvme_prp_dma_constructor()
1569 nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, in nvme_zalloc_queue_dma() argument
1573 ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; in nvme_zalloc_queue_dma()
1575 len = roundup(len, nvme->n_pagesize); in nvme_zalloc_queue_dma()
1577 if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) in nvme_zalloc_queue_dma()
1579 dev_err(nvme->n_dip, CE_WARN, in nvme_zalloc_queue_dma()
1585 dev_err(nvme->n_dip, CE_WARN, in nvme_zalloc_queue_dma()
1642 nvme_destroy_cq_array(nvme_t *nvme, uint_t start) in nvme_destroy_cq_array() argument
1646 for (i = start; i < nvme->n_cq_count; i++) in nvme_destroy_cq_array()
1647 if (nvme->n_cq[i] != NULL) in nvme_destroy_cq_array()
1648 nvme_free_cq(nvme->n_cq[i]); in nvme_destroy_cq_array()
1650 kmem_free(nvme->n_cq, sizeof (*nvme->n_cq) * nvme->n_cq_count); in nvme_destroy_cq_array()
1654 nvme_alloc_cq(nvme_t *nvme, uint32_t nentry, nvme_cq_t **cqp, uint16_t idx, in nvme_alloc_cq() argument
1661 DDI_INTR_PRI(nvme->n_intr_pri)); in nvme_alloc_cq()
1663 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), in nvme_alloc_cq()
1670 cq->ncq_hdbl = NVME_REG_CQHDBL(nvme, idx); in nvme_alloc_cq()
1676 ddi_driver_name(nvme->n_dip), ddi_get_instance(nvme->n_dip), idx); in nvme_alloc_cq()
1682 dev_err(nvme->n_dip, CE_WARN, "!failed to create cmd " in nvme_alloc_cq()
1704 nvme_create_cq_array(nvme_t *nvme, uint_t ncq, uint32_t nentry, uint_t nthr) in nvme_create_cq_array() argument
1709 ASSERT3U(ncq, >, nvme->n_cq_count); in nvme_create_cq_array()
1711 cq = nvme->n_cq; in nvme_create_cq_array()
1712 cq_count = nvme->n_cq_count; in nvme_create_cq_array()
1714 nvme->n_cq = kmem_zalloc(sizeof (*nvme->n_cq) * ncq, KM_SLEEP); in nvme_create_cq_array()
1715 nvme->n_cq_count = ncq; in nvme_create_cq_array()
1718 nvme->n_cq[i] = cq[i]; in nvme_create_cq_array()
1720 for (; i < nvme->n_cq_count; i++) in nvme_create_cq_array()
1721 if (nvme_alloc_cq(nvme, nentry, &nvme->n_cq[i], i, nthr) != in nvme_create_cq_array()
1731 nvme_destroy_cq_array(nvme, cq_count); in nvme_create_cq_array()
1735 nvme->n_cq_count = cq_count; in nvme_create_cq_array()
1736 nvme->n_cq = cq; in nvme_create_cq_array()
1742 nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, in nvme_alloc_qpair() argument
1749 DDI_INTR_PRI(nvme->n_intr_pri)); in nvme_alloc_qpair()
1757 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), in nvme_alloc_qpair()
1764 cq_idx = idx == 0 ? 0 : 1 + (idx - 1) % (nvme->n_cq_count - 1); in nvme_alloc_qpair()
1765 qp->nq_cq = nvme->n_cq[cq_idx]; in nvme_alloc_qpair()
1769 qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); in nvme_alloc_qpair()
1798 nvme_alloc_cmd(nvme_t *nvme, int kmflag) in nvme_alloc_cmd() argument
1804 cmd->nc_nvme = nvme; in nvme_alloc_cmd()
1811 nvme_alloc_admin_cmd(nvme_t *nvme, int kmflag) in nvme_alloc_admin_cmd() argument
1813 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, kmflag); in nvme_alloc_admin_cmd()
1818 DDI_INTR_PRI(nvme->n_intr_pri)); in nvme_alloc_admin_cmd()
1940 nvme_unqueue_cmd(nvme_t *nvme, nvme_qpair_t *qp, int cid) in nvme_unqueue_cmd() argument
1969 ASSERT3P(cmd->nc_nvme, ==, nvme); in nvme_unqueue_cmd()
1983 nvme_lost_cmd(nvme_t *nvme, nvme_cmd_t *cmd) in nvme_lost_cmd() argument
1989 nvme_qpair_t *qp = nvme->n_ioq[cmd->nc_sqid]; in nvme_lost_cmd()
1999 (void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); in nvme_lost_cmd()
2039 nvme_get_completed(nvme_t *nvme, nvme_cq_t *cq) in nvme_get_completed() argument
2054 qp = nvme->n_ioq[cqe->cqe_sqid]; in nvme_get_completed()
2057 cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid); in nvme_get_completed()
2068 dev_err(nvme->n_dip, CE_WARN, in nvme_get_completed()
2070 NVME_BUMP_STAT(nvme, unknown_cid); in nvme_get_completed()
2092 nvme_process_iocq(nvme_t *nvme, nvme_cq_t *cq) in nvme_process_iocq() argument
2100 dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s", in nvme_process_iocq()
2105 while ((cmd = nvme_get_completed(nvme, cq)) != NULL) { in nvme_process_iocq()
2141 nvme_put32(nvme, cq->ncq_hdbl, head.r); in nvme_process_iocq()
2150 nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) in nvme_retrieve_cmd() argument
2158 dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s", in nvme_retrieve_cmd()
2163 if ((cmd = nvme_get_completed(nvme, cq)) != NULL) { in nvme_retrieve_cmd()
2165 nvme_put32(nvme, cq->ncq_hdbl, head.r); in nvme_retrieve_cmd()
2609 nvme_t *nvme = cmd->nc_nvme; in nvme_check_cmd_status_ioctl() local
2611 if (nvme->n_dead) { in nvme_check_cmd_status_ioctl()
2612 return (nvme_ioctl_error(ioc, nvme->n_dead_status, 0, 0)); in nvme_check_cmd_status_ioctl()
2638 nvme_t *nvme = cmd->nc_nvme; in nvme_abort_cmd() local
2639 nvme_cmd_t *abort_cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_abort_cmd()
2643 sema_p(&nvme->n_abort_sema); in nvme_abort_cmd()
2665 sema_v(&nvme->n_abort_sema); in nvme_abort_cmd()
2685 dev_err(nvme->n_dip, CE_WARN, in nvme_abort_cmd()
2688 NVME_BUMP_STAT(nvme, abort_timeout); in nvme_abort_cmd()
2691 dev_err(nvme->n_dip, CE_WARN, in nvme_abort_cmd()
2697 NVME_BUMP_STAT(nvme, abort_failed); in nvme_abort_cmd()
2701 dev_err(nvme->n_dip, CE_WARN, in nvme_abort_cmd()
2707 NVME_BUMP_STAT(nvme, abort_successful); in nvme_abort_cmd()
2709 NVME_BUMP_STAT(nvme, abort_unsuccessful); in nvme_abort_cmd()
2731 nvme_t *nvme = cmd->nc_nvme; in nvme_wait_cmd() local
2759 nvme_admin_stat_cmd(nvme, cmd); in nvme_wait_cmd()
2768 csts.r = nvme_get32(nvme, NVME_REG_CSTS); in nvme_wait_cmd()
2769 dev_err(nvme->n_dip, CE_WARN, "!command %d/%d timeout, " in nvme_wait_cmd()
2772 NVME_BUMP_STAT(nvme, cmd_timeout); in nvme_wait_cmd()
2780 nvme_check_regs_hdl(nvme) || in nvme_wait_cmd()
2784 nvme_lost_cmd(nvme, cmd); in nvme_wait_cmd()
2807 nvme_lost_cmd(nvme, cmd); in nvme_wait_cmd()
2827 nvme_t *nvme = cmd->nc_nvme; in nvme_async_event_task() local
2864 nvme->n_async_event_supported = B_FALSE; in nvme_async_event_task()
2875 nvme_submit_admin_cmd(nvme->n_adminq, cmd, NULL); in nvme_async_event_task()
2881 if (!nvme_get_logpage_int(nvme, B_FALSE, in nvme_async_event_task()
2887 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " in nvme_async_event_task()
2890 NVME_BUMP_STAT(nvme, wrong_logpage); in nvme_async_event_task()
2896 dev_err(nvme->n_dip, CE_PANIC, "programming error: " in nvme_async_event_task()
2901 dev_err(nvme->n_dip, CE_PANIC, "programming error: " in nvme_async_event_task()
2906 dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); in nvme_async_event_task()
2907 nvme_ctrl_mark_dead(nvme, B_FALSE); in nvme_async_event_task()
2908 NVME_BUMP_STAT(nvme, diagfail_event); in nvme_async_event_task()
2912 dev_err(nvme->n_dip, CE_WARN, "!persistent internal " in nvme_async_event_task()
2914 nvme_ctrl_mark_dead(nvme, B_FALSE); in nvme_async_event_task()
2915 NVME_BUMP_STAT(nvme, persistent_event); in nvme_async_event_task()
2919 dev_err(nvme->n_dip, CE_WARN, "!transient internal " in nvme_async_event_task()
2922 NVME_BUMP_STAT(nvme, transient_event); in nvme_async_event_task()
2926 dev_err(nvme->n_dip, CE_WARN, in nvme_async_event_task()
2928 NVME_BUMP_STAT(nvme, fw_load_event); in nvme_async_event_task()
2935 if (!nvme_get_logpage_int(nvme, B_FALSE, in nvme_async_event_task()
2941 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " in nvme_async_event_task()
2944 NVME_BUMP_STAT(nvme, wrong_logpage); in nvme_async_event_task()
2950 dev_err(nvme->n_dip, CE_WARN, in nvme_async_event_task()
2953 NVME_BUMP_STAT(nvme, reliability_event); in nvme_async_event_task()
2957 dev_err(nvme->n_dip, CE_WARN, in nvme_async_event_task()
2960 NVME_BUMP_STAT(nvme, temperature_event); in nvme_async_event_task()
2964 dev_err(nvme->n_dip, CE_WARN, in nvme_async_event_task()
2967 NVME_BUMP_STAT(nvme, spare_event); in nvme_async_event_task()
2976 dev_err(nvme->n_dip, CE_WARN, in nvme_async_event_task()
2980 NVME_BUMP_STAT(nvme, wrong_logpage); in nvme_async_event_task()
2984 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
2987 NVME_BUMP_STAT(nvme, notice_event); in nvme_async_event_task()
2989 if (!nvme_get_logpage_int(nvme, B_FALSE, in nvme_async_event_task()
2996 dev_err(nvme->n_dip, CE_CONT, in nvme_async_event_task()
3002 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_async_event_task()
3010 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
3012 ddi_get_instance(nvme->n_dip), nsid); in nvme_async_event_task()
3014 if (nvme_init_ns(nvme, nsid) != DDI_SUCCESS) in nvme_async_event_task()
3017 ns = nvme_nsid2ns(nvme, nsid); in nvme_async_event_task()
3021 nvme_mgmt_bd_start(nvme); in nvme_async_event_task()
3023 nvme_mgmt_bd_end(nvme); in nvme_async_event_task()
3025 nvme_mgmt_unlock(nvme); in nvme_async_event_task()
3030 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
3033 NVME_BUMP_STAT(nvme, notice_event); in nvme_async_event_task()
3037 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
3040 NVME_BUMP_STAT(nvme, notice_event); in nvme_async_event_task()
3044 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
3047 NVME_BUMP_STAT(nvme, notice_event); in nvme_async_event_task()
3051 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
3054 NVME_BUMP_STAT(nvme, notice_event); in nvme_async_event_task()
3058 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
3061 NVME_BUMP_STAT(nvme, notice_event); in nvme_async_event_task()
3065 dev_err(nvme->n_dip, CE_NOTE, in nvme_async_event_task()
3068 NVME_BUMP_STAT(nvme, notice_event); in nvme_async_event_task()
3072 dev_err(nvme->n_dip, CE_WARN, in nvme_async_event_task()
3076 NVME_BUMP_STAT(nvme, unknown_event); in nvme_async_event_task()
3082 dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " in nvme_async_event_task()
3085 NVME_BUMP_STAT(nvme, vendor_event); in nvme_async_event_task()
3089 dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " in nvme_async_event_task()
3092 NVME_BUMP_STAT(nvme, unknown_event); in nvme_async_event_task()
3129 nvme_async_event(nvme_t *nvme) in nvme_async_event() argument
3133 cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_async_event()
3139 nvme_submit_admin_cmd(nvme->n_adminq, cmd, NULL); in nvme_async_event()
3148 nvme_no_blkdev_attached(nvme_t *nvme, uint32_t nsid) in nvme_no_blkdev_attached() argument
3150 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_no_blkdev_attached()
3154 nvme_namespace_t *ns = nvme_nsid2ns(nvme, nsid); in nvme_no_blkdev_attached()
3158 for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) { in nvme_no_blkdev_attached()
3159 nvme_namespace_t *ns = nvme_nsid2ns(nvme, i); in nvme_no_blkdev_attached()
3170 nvme_format_nvm(nvme_t *nvme, nvme_ioctl_format_t *ioc) in nvme_format_nvm() argument
3172 nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_format_nvm()
3201 dev_err(nvme->n_dip, CE_WARN, in nvme_format_nvm()
3219 nvme_get_logpage(nvme_t *nvme, boolean_t user, nvme_ioctl_get_logpage_t *log, in nvme_get_logpage() argument
3222 nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_get_logpage()
3269 if (nvme_zalloc_dma(nvme, log->nigl_len, DDI_DMA_READ, in nvme_get_logpage()
3270 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { in nvme_get_logpage()
3271 dev_err(nvme->n_dip, CE_WARN, in nvme_get_logpage()
3287 dev_err(nvme->n_dip, CE_WARN, in nvme_get_logpage()
3313 nvme_get_logpage_int(nvme_t *nvme, boolean_t user, void **buf, size_t *bufsize, in nvme_get_logpage_int() argument
3334 data.vcd_vers = &nvme->n_version; in nvme_get_logpage_int()
3335 data.vcd_id = nvme->n_idctl; in nvme_get_logpage_int()
3351 bret = nvme_get_logpage(nvme, user, &log, buf); in nvme_get_logpage_int()
3361 nvme_identify(nvme_t *nvme, boolean_t user, nvme_ioctl_identify_t *ioc, in nvme_identify() argument
3364 nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_identify()
3382 if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, in nvme_identify()
3383 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { in nvme_identify()
3384 dev_err(nvme->n_dip, CE_WARN, in nvme_identify()
3392 dev_err(nvme->n_dip, CE_WARN, in nvme_identify()
3394 NVME_BUMP_STAT(nvme, too_many_cookies); in nvme_identify()
3414 dev_err(nvme->n_dip, CE_WARN, in nvme_identify()
3432 nvme_identify_int(nvme_t *nvme, uint32_t nsid, uint8_t cns, void **buf) in nvme_identify_int() argument
3440 return (nvme_identify(nvme, B_FALSE, &id, buf)); in nvme_identify_int()
3444 nvme_set_features(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t feature, in nvme_set_features() argument
3448 nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_set_features()
3464 if (!nvme->n_write_cache_present) in nvme_set_features()
3478 dev_err(nvme->n_dip, CE_WARN, in nvme_set_features()
3493 nvme_write_cache_set(nvme_t *nvme, boolean_t enable) in nvme_write_cache_set() argument
3508 return (nvme_set_features(nvme, B_TRUE, 0, NVME_FEAT_WRITE_CACHE, in nvme_write_cache_set()
3513 nvme_set_nqueues(nvme_t *nvme) in nvme_set_nqueues() argument
3521 if (nvme->n_completion_queues == -1) in nvme_set_nqueues()
3522 nvme->n_completion_queues = nvme->n_intr_cnt; in nvme_set_nqueues()
3528 nvme->n_completion_queues = MIN(nvme->n_completion_queues, in nvme_set_nqueues()
3529 nvme->n_intr_cnt); in nvme_set_nqueues()
3534 if (nvme->n_submission_queues == -1) in nvme_set_nqueues()
3535 nvme->n_submission_queues = nvme->n_completion_queues; in nvme_set_nqueues()
3541 nvme->n_completion_queues = MIN(nvme->n_completion_queues, in nvme_set_nqueues()
3542 nvme->n_submission_queues); in nvme_set_nqueues()
3544 ASSERT(nvme->n_submission_queues > 0); in nvme_set_nqueues()
3545 ASSERT(nvme->n_completion_queues > 0); in nvme_set_nqueues()
3547 nq.b.nq_nsq = nvme->n_submission_queues - 1; in nvme_set_nqueues()
3548 nq.b.nq_ncq = nvme->n_completion_queues - 1; in nvme_set_nqueues()
3550 ret = nvme_set_features(nvme, B_FALSE, 0, NVME_FEAT_NQUEUES, nq.r, in nvme_set_nqueues()
3557 nvme->n_submission_queues = MIN(nvme->n_submission_queues, in nvme_set_nqueues()
3559 nvme->n_completion_queues = MIN(nvme->n_completion_queues, in nvme_set_nqueues()
3567 nvme_create_completion_queue(nvme_t *nvme, nvme_cq_t *cq) in nvme_create_completion_queue() argument
3569 nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_create_completion_queue()
3579 c_dw11.b.cq_iv = cq->ncq_id % nvme->n_intr_cnt; in nvme_create_completion_queue()
3591 dev_err(nvme->n_dip, CE_WARN, in nvme_create_completion_queue()
3602 nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) in nvme_create_io_qpair() argument
3616 nvme_create_completion_queue(nvme, cq) != DDI_SUCCESS) in nvme_create_io_qpair()
3625 cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_create_io_qpair()
3636 dev_err(nvme->n_dip, CE_WARN, in nvme_create_io_qpair()
3647 nvme_reset(nvme_t *nvme, boolean_t quiesce) in nvme_reset() argument
3656 if (nvme_ctrl_is_gone(nvme)) { in nvme_reset()
3660 nvme_put32(nvme, NVME_REG_CC, 0); in nvme_reset()
3662 csts.r = nvme_get32(nvme, NVME_REG_CSTS); in nvme_reset()
3664 nvme_put32(nvme, NVME_REG_CC, 0); in nvme_reset()
3675 for (i = 0; i < nvme->n_timeout * 10; i++) { in nvme_reset()
3676 csts.r = nvme_get32(nvme, NVME_REG_CSTS); in nvme_reset()
3693 nvme_put32(nvme, NVME_REG_AQA, 0); in nvme_reset()
3694 nvme_put32(nvme, NVME_REG_ASQ, 0); in nvme_reset()
3695 nvme_put32(nvme, NVME_REG_ACQ, 0); in nvme_reset()
3697 csts.r = nvme_get32(nvme, NVME_REG_CSTS); in nvme_reset()
3702 nvme_shutdown(nvme_t *nvme, boolean_t quiesce) in nvme_shutdown() argument
3712 if (nvme_ctrl_is_gone(nvme)) { in nvme_shutdown()
3716 cc.r = nvme_get32(nvme, NVME_REG_CC); in nvme_shutdown()
3718 nvme_put32(nvme, NVME_REG_CC, cc.r); in nvme_shutdown()
3721 csts.r = nvme_get32(nvme, NVME_REG_CSTS); in nvme_shutdown()
3749 nvme_config_min_block_size(nvme_t *nvme, char *model, char *val) in nvme_config_min_block_size() argument
3767 nvme->n_min_block_size = bsize; in nvme_config_min_block_size()
3771 dev_err(nvme->n_dip, CE_WARN, in nvme_config_min_block_size()
3775 nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE; in nvme_config_min_block_size()
3779 nvme_config_boolean(nvme_t *nvme, char *model, char *name, char *val, in nvme_config_boolean() argument
3789 dev_err(nvme->n_dip, CE_WARN, in nvme_config_boolean()
3795 nvme_config_list(nvme_t *nvme) in nvme_config_list() argument
3806 rv = ddi_prop_lookup_string_array(DDI_DEV_T_ANY, nvme->n_dip, in nvme_config_list()
3811 dev_err(nvme->n_dip, CE_WARN, in nvme_config_list()
3819 dev_err(nvme->n_dip, CE_WARN, "!nvme-config-list: must be " in nvme_config_list()
3830 id_model_len = nvme_strlen(nvme->n_idctl->id_model, in nvme_config_list()
3831 sizeof (nvme->n_idctl->id_model)); in nvme_config_list()
3836 if (strncmp(model, nvme->n_idctl->id_model, id_model_len) != 0) in nvme_config_list()
3839 id_fwrev_len = nvme_strlen(nvme->n_idctl->id_fwrev, in nvme_config_list()
3840 sizeof (nvme->n_idctl->id_fwrev)); in nvme_config_list()
3852 if (strncmp(fwr, nvme->n_idctl->id_fwrev, in nvme_config_list()
3871 dev_err(nvme->n_dip, CE_WARN, in nvme_config_list()
3886 nvme_config_boolean(nvme, model, name, val, in nvme_config_list()
3887 &nvme->n_ignore_unknown_vendor_status); in nvme_config_list()
3889 nvme_config_min_block_size(nvme, model, val); in nvme_config_list()
3891 nvme_config_boolean(nvme, model, name, val, in nvme_config_list()
3892 &nvme->n_write_cache_enabled); in nvme_config_list()
3897 dev_err(nvme->n_dip, CE_WARN, in nvme_config_list()
3909 nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) in nvme_prepare_devid() argument
3919 char model[sizeof (nvme->n_idctl->id_model) + 1]; in nvme_prepare_devid()
3920 char serial[sizeof (nvme->n_idctl->id_serial) + 1]; in nvme_prepare_devid()
3922 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); in nvme_prepare_devid()
3923 bcopy(nvme->n_idctl->id_serial, serial, in nvme_prepare_devid()
3924 sizeof (nvme->n_idctl->id_serial)); in nvme_prepare_devid()
3926 model[sizeof (nvme->n_idctl->id_model)] = '\0'; in nvme_prepare_devid()
3927 serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; in nvme_prepare_devid()
3929 nvme_nsid2ns(nvme, nsid)->ns_devid = kmem_asprintf("%4X-%s-%s-%X", in nvme_prepare_devid()
3930 nvme->n_idctl->id_vid, model, serial, nsid); in nvme_prepare_devid()
3934 nvme_update_nsid_list(nvme_t *nvme, int cns) in nvme_update_nsid_list() argument
3942 if (nvme_identify_int(nvme, 0, cns, (void **)&nslist)) in nvme_update_nsid_list()
3949 nvme_nsid2ns(nvme_t *nvme, uint32_t nsid) in nvme_nsid2ns() argument
3952 ASSERT3U(nsid, <=, nvme->n_namespace_count); in nvme_nsid2ns()
3953 return (&nvme->n_ns[nsid - 1]); in nvme_nsid2ns()
3959 nvme_t *nvme = ns->ns_nvme; in nvme_allocated_ns() local
3962 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_allocated_ns()
3967 if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 2) && in nvme_allocated_ns()
3968 nvme->n_idctl->id_oacs.oa_nsmgmt != 0) { in nvme_allocated_ns()
3969 nvme_identify_nsid_list_t *nslist = nvme_update_nsid_list(nvme, in nvme_allocated_ns()
4002 nvme_t *nvme = ns->ns_nvme; in nvme_active_ns() local
4006 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_active_ns()
4011 if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) { in nvme_active_ns()
4012 nvme_identify_nsid_list_t *nslist = nvme_update_nsid_list(nvme, in nvme_active_ns()
4051 nvme_init_ns(nvme_t *nvme, uint32_t nsid) in nvme_init_ns() argument
4053 nvme_namespace_t *ns = nvme_nsid2ns(nvme, nsid); in nvme_init_ns()
4057 ns->ns_nvme = nvme; in nvme_init_ns()
4059 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_init_ns()
4067 if (!nvme_identify_int(nvme, nsid, NVME_IDENTIFY_NSID, in nvme_init_ns()
4069 dev_err(nvme->n_dip, CE_WARN, in nvme_init_ns()
4111 if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) in nvme_init_ns()
4117 if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 2)) in nvme_init_ns()
4122 nvme_prepare_devid(nvme, ns->ns_id); in nvme_init_ns()
4142 if (ns->ns_best_block_size < nvme->n_min_block_size) in nvme_init_ns()
4143 ns->ns_best_block_size = nvme->n_min_block_size; in nvme_init_ns()
4153 dev_err(nvme->n_dip, CE_WARN, in nvme_init_ns()
4160 dev_err(nvme->n_dip, CE_WARN, in nvme_init_ns()
4174 dev_err(nvme->n_dip, CE_PANIC, "namespace %u state " in nvme_init_ns()
4188 nvme->n_namespaces_attachable--; in nvme_init_ns()
4193 nvme->n_namespaces_attachable++; in nvme_init_ns()
4200 nvme_bd_attach_ns(nvme_t *nvme, nvme_ioctl_common_t *com) in nvme_bd_attach_ns() argument
4202 nvme_namespace_t *ns = nvme_nsid2ns(nvme, com->nioc_nsid); in nvme_bd_attach_ns()
4205 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_bd_attach_ns()
4214 if (!nvme->n_idctl->id_oncs.on_dset_mgmt) in nvme_bd_attach_ns()
4217 ns->ns_bd_hdl = bd_alloc_handle(ns, &ops, &nvme->n_prp_dma_attr, in nvme_bd_attach_ns()
4221 dev_err(nvme->n_dip, CE_WARN, "!Failed to get blkdev " in nvme_bd_attach_ns()
4228 nvme_mgmt_bd_start(nvme); in nvme_bd_attach_ns()
4229 ret = bd_attach_handle(nvme->n_dip, ns->ns_bd_hdl); in nvme_bd_attach_ns()
4230 nvme_mgmt_bd_end(nvme); in nvme_bd_attach_ns()
4242 nvme_bd_detach_ns(nvme_t *nvme, nvme_ioctl_common_t *com) in nvme_bd_detach_ns() argument
4244 nvme_namespace_t *ns = nvme_nsid2ns(nvme, com->nioc_nsid); in nvme_bd_detach_ns()
4247 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_bd_detach_ns()
4253 nvme_mgmt_bd_start(nvme); in nvme_bd_detach_ns()
4256 nvme_mgmt_bd_end(nvme); in nvme_bd_detach_ns()
4273 nvme_rescan_ns(nvme_t *nvme, uint32_t nsid) in nvme_rescan_ns() argument
4275 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_rescan_ns()
4279 nvme_namespace_t *ns = nvme_nsid2ns(nvme, nsid); in nvme_rescan_ns()
4282 (void) nvme_init_ns(nvme, nsid); in nvme_rescan_ns()
4286 for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) { in nvme_rescan_ns()
4287 nvme_namespace_t *ns = nvme_nsid2ns(nvme, i); in nvme_rescan_ns()
4290 (void) nvme_init_ns(nvme, i); in nvme_rescan_ns()
4305 nvme_detect_quirks(nvme_t *nvme) in nvme_detect_quirks() argument
4310 if (nqt->nq_vendor_id == nvme->n_vendor_id && in nvme_detect_quirks()
4311 nqt->nq_device_id == nvme->n_device_id) { in nvme_detect_quirks()
4312 nvme->n_quirks = nqt->nq_quirks; in nvme_detect_quirks()
4319 nvme_init(nvme_t *nvme) in nvme_init() argument
4331 char model[sizeof (nvme->n_idctl->id_model) + 1]; in nvme_init()
4336 vs.r = nvme_get32(nvme, NVME_REG_VS); in nvme_init()
4337 nvme->n_version.v_major = vs.b.vs_mjr; in nvme_init()
4338 nvme->n_version.v_minor = vs.b.vs_mnr; in nvme_init()
4339 dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d\n", in nvme_init()
4340 nvme->n_version.v_major, nvme->n_version.v_minor); in nvme_init()
4342 if (nvme->n_version.v_major > nvme_version_major) { in nvme_init()
4343 dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.x", in nvme_init()
4345 if (nvme->n_strict_version) in nvme_init()
4350 cap.r = nvme_get64(nvme, NVME_REG_CAP); in nvme_init()
4353 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4358 nvme->n_nssr_supported = cap.b.cap_nssrs; in nvme_init()
4359 nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; in nvme_init()
4360 nvme->n_timeout = cap.b.cap_to; in nvme_init()
4361 nvme->n_arbitration_mechanisms = cap.b.cap_ams; in nvme_init()
4362 nvme->n_cont_queues_reqd = cap.b.cap_cqr; in nvme_init()
4363 nvme->n_max_queue_entries = cap.b.cap_mqes + 1; in nvme_init()
4370 nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), in nvme_init()
4372 nvme->n_pagesize = 1UL << (nvme->n_pageshift); in nvme_init()
4377 nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; in nvme_init()
4378 nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; in nvme_init()
4384 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; in nvme_init()
4385 nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; in nvme_init()
4386 nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; in nvme_init()
4387 nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; in nvme_init()
4392 if (nvme_reset(nvme, B_FALSE) == B_FALSE) { in nvme_init()
4393 dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); in nvme_init()
4394 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); in nvme_init()
4395 nvme->n_dead = B_TRUE; in nvme_init()
4403 if (nvme_create_cq_array(nvme, 1, nvme->n_admin_queue_len, 4) != in nvme_init()
4405 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4412 if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) in nvme_init()
4414 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4418 nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); in nvme_init()
4419 nvme->n_ioq[0] = nvme->n_adminq; in nvme_init()
4421 if (nvme->n_quirks & NVME_QUIRK_START_CID) in nvme_init()
4422 nvme->n_adminq->nq_next_cmd++; in nvme_init()
4424 nvme->n_progress |= NVME_ADMIN_QUEUE; in nvme_init()
4426 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, in nvme_init()
4427 "admin-queue-len", nvme->n_admin_queue_len); in nvme_init()
4429 aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; in nvme_init()
4430 asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; in nvme_init()
4431 acq = nvme->n_adminq->nq_cq->ncq_dma->nd_cookie.dmac_laddress; in nvme_init()
4433 ASSERT((asq & (nvme->n_pagesize - 1)) == 0); in nvme_init()
4434 ASSERT((acq & (nvme->n_pagesize - 1)) == 0); in nvme_init()
4436 nvme_put32(nvme, NVME_REG_AQA, aqa.r); in nvme_init()
4437 nvme_put64(nvme, NVME_REG_ASQ, asq); in nvme_init()
4438 nvme_put64(nvme, NVME_REG_ACQ, acq); in nvme_init()
4442 cc.b.cc_mps = nvme->n_pageshift - 12; in nvme_init()
4448 nvme_put32(nvme, NVME_REG_CC, cc.r); in nvme_init()
4453 csts.r = nvme_get32(nvme, NVME_REG_CSTS); in nvme_init()
4455 for (i = 0; i != nvme->n_timeout * 10; i++) { in nvme_init()
4457 csts.r = nvme_get32(nvme, NVME_REG_CSTS); in nvme_init()
4460 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4462 ddi_fm_service_impact(nvme->n_dip, in nvme_init()
4464 nvme->n_dead = B_TRUE; in nvme_init()
4474 dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); in nvme_init()
4475 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); in nvme_init()
4476 nvme->n_dead = B_TRUE; in nvme_init()
4484 sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); in nvme_init()
4489 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) in nvme_init()
4491 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) in nvme_init()
4493 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) in nvme_init()
4495 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4504 nvme->n_dead_status = NVME_IOCTL_E_CTRL_DEAD; in nvme_init()
4509 if (!nvme_identify_int(nvme, 0, NVME_IDENTIFY_CTRL, in nvme_init()
4510 (void **)&nvme->n_idctl)) { in nvme_init()
4511 dev_err(nvme->n_dip, CE_WARN, "!failed to identify controller"); in nvme_init()
4518 nvme_config_list(nvme); in nvme_init()
4523 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); in nvme_init()
4524 model[sizeof (nvme->n_idctl->id_model)] = '\0'; in nvme_init()
4528 nvme->n_vendor = strdup("NVMe"); in nvme_init()
4530 nvme->n_vendor = strdup(vendor); in nvme_init()
4532 nvme->n_product = strdup(product); in nvme_init()
4537 nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, in nvme_init()
4538 MIN(nvme->n_admin_queue_len / 10, in nvme_init()
4539 MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); in nvme_init()
4541 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, in nvme_init()
4542 "async-event-limit", nvme->n_async_event_limit); in nvme_init()
4544 nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; in nvme_init()
4552 sema_destroy(&nvme->n_abort_sema); in nvme_init()
4553 sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, in nvme_init()
4556 nvme->n_progress |= NVME_CTRL_LIMITS; in nvme_init()
4558 if (nvme->n_idctl->id_mdts == 0) in nvme_init()
4559 nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; in nvme_init()
4561 nvme->n_max_data_transfer_size = in nvme_init()
4562 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); in nvme_init()
4564 nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; in nvme_init()
4573 nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, in nvme_init()
4574 (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); in nvme_init()
4576 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; in nvme_init()
4583 if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || in nvme_init()
4584 ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || in nvme_init()
4585 ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || in nvme_init()
4586 ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) in nvme_init()
4594 nvme->n_write_cache_present = in nvme_init()
4595 nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE; in nvme_init()
4597 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, in nvme_init()
4599 nvme->n_write_cache_present ? 1 : 0); in nvme_init()
4601 if (!nvme->n_write_cache_present) { in nvme_init()
4602 nvme->n_write_cache_enabled = B_FALSE; in nvme_init()
4603 } else if (nvme_write_cache_set(nvme, nvme->n_write_cache_enabled) in nvme_init()
4605 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4607 nvme->n_write_cache_enabled ? "en" : "dis"); in nvme_init()
4611 nvme->n_write_cache_enabled = B_TRUE; in nvme_init()
4614 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, in nvme_init()
4616 nvme->n_write_cache_enabled ? 1 : 0); in nvme_init()
4621 nvme->n_namespace_count = nvme->n_idctl->id_nn; in nvme_init()
4623 if (nvme->n_namespace_count == 0) { in nvme_init()
4624 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4629 nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * in nvme_init()
4630 nvme->n_namespace_count, KM_SLEEP); in nvme_init()
4636 if (nvme_ctrl_atleast(nvme, &nvme_vers_1v2) && in nvme_init()
4637 nvme->n_idctl->id_oacs.oa_nsmgmt != 0) { in nvme_init()
4643 if (!nvme_identify_int(nvme, nsid, NVME_IDENTIFY_NSID, in nvme_init()
4644 (void **)&nvme->n_idcomns)) { in nvme_init()
4645 dev_err(nvme->n_dip, CE_WARN, "!failed to identify common " in nvme_init()
4653 if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) in nvme_init()
4655 nvme_release_interrupts(nvme); in nvme_init()
4659 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, in nvme_init()
4661 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, in nvme_init()
4663 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4673 if (nvme_set_nqueues(nvme) != 0) { in nvme_init()
4674 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4676 nvme->n_intr_cnt); in nvme_init()
4683 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); in nvme_init()
4684 nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * in nvme_init()
4685 (nvme->n_submission_queues + 1), KM_SLEEP); in nvme_init()
4686 nvme->n_ioq[0] = nvme->n_adminq; in nvme_init()
4692 ASSERT(nvme->n_submission_queues >= nvme->n_completion_queues); in nvme_init()
4694 nvme->n_ioq_count = nvme->n_submission_queues; in nvme_init()
4696 nvme->n_io_squeue_len = in nvme_init()
4697 MIN(nvme->n_io_squeue_len, nvme->n_max_queue_entries); in nvme_init()
4699 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-squeue-len", in nvme_init()
4700 nvme->n_io_squeue_len); in nvme_init()
4708 if (nvme->n_submission_queues == nvme->n_completion_queues) in nvme_init()
4709 nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len, in nvme_init()
4710 nvme->n_io_squeue_len); in nvme_init()
4712 nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len, in nvme_init()
4713 nvme->n_max_queue_entries); in nvme_init()
4715 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-cqueue-len", in nvme_init()
4716 nvme->n_io_cqueue_len); in nvme_init()
4723 tq_threads = MIN(UINT16_MAX, ncpus) / nvme->n_completion_queues; in nvme_init()
4731 if (nvme_create_cq_array(nvme, nvme->n_completion_queues + 1, in nvme_init()
4732 nvme->n_io_cqueue_len, tq_threads) != DDI_SUCCESS) { in nvme_init()
4733 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4742 if (nvme->n_completion_queues + 1 < nvme->n_intr_cnt) { in nvme_init()
4743 nvme_release_interrupts(nvme); in nvme_init()
4745 if (nvme_setup_interrupts(nvme, nvme->n_intr_type, in nvme_init()
4746 nvme->n_completion_queues + 1) != DDI_SUCCESS) { in nvme_init()
4747 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4757 for (i = 1; i != nvme->n_ioq_count + 1; i++) { in nvme_init()
4758 if (nvme_alloc_qpair(nvme, nvme->n_io_squeue_len, in nvme_init()
4759 &nvme->n_ioq[i], i) != DDI_SUCCESS) { in nvme_init()
4760 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4765 if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) != 0) { in nvme_init()
4766 dev_err(nvme->n_dip, CE_WARN, in nvme_init()
4775 (void) nvme_reset(nvme, B_FALSE); in nvme_init()
4782 nvme_t *nvme = (nvme_t *)arg1; in nvme_intr() local
4787 if (inum >= nvme->n_intr_cnt) in nvme_intr()
4790 if (nvme->n_dead) { in nvme_intr()
4791 return (nvme->n_intr_type == DDI_INTR_TYPE_FIXED ? in nvme_intr()
4801 qnum < nvme->n_cq_count && nvme->n_cq[qnum] != NULL; in nvme_intr()
4802 qnum += nvme->n_intr_cnt) { in nvme_intr()
4803 ccnt += nvme_process_iocq(nvme, nvme->n_cq[qnum]); in nvme_intr()
4810 nvme_release_interrupts(nvme_t *nvme) in nvme_release_interrupts() argument
4814 for (i = 0; i < nvme->n_intr_cnt; i++) { in nvme_release_interrupts()
4815 if (nvme->n_inth[i] == NULL) in nvme_release_interrupts()
4818 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) in nvme_release_interrupts()
4819 (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); in nvme_release_interrupts()
4821 (void) ddi_intr_disable(nvme->n_inth[i]); in nvme_release_interrupts()
4823 (void) ddi_intr_remove_handler(nvme->n_inth[i]); in nvme_release_interrupts()
4824 (void) ddi_intr_free(nvme->n_inth[i]); in nvme_release_interrupts()
4827 kmem_free(nvme->n_inth, nvme->n_inth_sz); in nvme_release_interrupts()
4828 nvme->n_inth = NULL; in nvme_release_interrupts()
4829 nvme->n_inth_sz = 0; in nvme_release_interrupts()
4831 nvme->n_progress &= ~NVME_INTERRUPTS; in nvme_release_interrupts()
4835 nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) in nvme_setup_interrupts() argument
4841 if (nvme->n_intr_types == 0) { in nvme_setup_interrupts()
4842 ret = ddi_intr_get_supported_types(nvme->n_dip, in nvme_setup_interrupts()
4843 &nvme->n_intr_types); in nvme_setup_interrupts()
4845 dev_err(nvme->n_dip, CE_WARN, in nvme_setup_interrupts()
4852 nvme->n_intr_types &= ~DDI_INTR_TYPE_MSIX; in nvme_setup_interrupts()
4856 if ((nvme->n_intr_types & intr_type) == 0) in nvme_setup_interrupts()
4859 ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); in nvme_setup_interrupts()
4861 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", in nvme_setup_interrupts()
4866 ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); in nvme_setup_interrupts()
4868 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", in nvme_setup_interrupts()
4877 nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; in nvme_setup_interrupts()
4878 nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); in nvme_setup_interrupts()
4880 ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, in nvme_setup_interrupts()
4883 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", in nvme_setup_interrupts()
4888 nvme->n_intr_cnt = count; in nvme_setup_interrupts()
4890 ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); in nvme_setup_interrupts()
4892 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", in nvme_setup_interrupts()
4898 ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, in nvme_setup_interrupts()
4899 (void *)nvme, (void *)(uintptr_t)i); in nvme_setup_interrupts()
4901 dev_err(nvme->n_dip, CE_WARN, in nvme_setup_interrupts()
4907 (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); in nvme_setup_interrupts()
4910 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) in nvme_setup_interrupts()
4911 ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); in nvme_setup_interrupts()
4913 ret = ddi_intr_enable(nvme->n_inth[i]); in nvme_setup_interrupts()
4916 dev_err(nvme->n_dip, CE_WARN, in nvme_setup_interrupts()
4922 nvme->n_intr_type = intr_type; in nvme_setup_interrupts()
4924 nvme->n_progress |= NVME_INTERRUPTS; in nvme_setup_interrupts()
4929 nvme_release_interrupts(nvme); in nvme_setup_interrupts()
4947 nvme_t *nvme = a; in nvme_remove_callback() local
4949 nvme_ctrl_mark_dead(nvme, B_TRUE); in nvme_remove_callback()
4955 for (uint_t i = 0; i < nvme->n_ioq_count + 1; i++) { in nvme_remove_callback()
4956 nvme_qpair_t *qp = nvme->n_ioq[i]; in nvme_remove_callback()
4974 u_cmd = nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); in nvme_remove_callback()
5033 nvme_t *nvme; in nvme_attach() local
5047 nvme = ddi_get_soft_state(nvme_state, instance); in nvme_attach()
5048 ddi_set_driver_private(dip, nvme); in nvme_attach()
5049 nvme->n_dip = dip; in nvme_attach()
5054 if (pci_config_setup(dip, &nvme->n_pcicfg_handle) != DDI_SUCCESS) { in nvme_attach()
5058 nvme->n_progress |= NVME_PCI_CONFIG; in nvme_attach()
5063 nvme->n_vendor_id = in nvme_attach()
5064 pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_VENID); in nvme_attach()
5065 nvme->n_device_id = in nvme_attach()
5066 pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_DEVID); in nvme_attach()
5067 nvme->n_revision_id = in nvme_attach()
5068 pci_config_get8(nvme->n_pcicfg_handle, PCI_CONF_REVID); in nvme_attach()
5069 nvme->n_subsystem_device_id = in nvme_attach()
5070 pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_SUBSYSID); in nvme_attach()
5071 nvme->n_subsystem_vendor_id = in nvme_attach()
5072 pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_SUBVENID); in nvme_attach()
5074 nvme_detect_quirks(nvme); in nvme_attach()
5085 if (ddi_get_eventcookie(nvme->n_dip, DDI_DEVI_REMOVE_EVENT, in nvme_attach()
5086 &nvme->n_rm_cookie) == DDI_SUCCESS) { in nvme_attach()
5087 if (ddi_add_event_handler(nvme->n_dip, nvme->n_rm_cookie, in nvme_attach()
5088 nvme_remove_callback, nvme, &nvme->n_ev_rm_cb_id) != in nvme_attach()
5093 nvme->n_ev_rm_cb_id = NULL; in nvme_attach()
5096 mutex_init(&nvme->n_minor_mutex, NULL, MUTEX_DRIVER, NULL); in nvme_attach()
5097 nvme->n_progress |= NVME_MUTEX_INIT; in nvme_attach()
5099 nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5101 nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, in nvme_attach()
5104 nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5106 nvme->n_io_squeue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5112 nvme->n_io_cqueue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5114 nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5117 nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5120 nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5123 nvme->n_submission_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5125 nvme->n_completion_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip, in nvme_attach()
5128 if (!ISP2(nvme->n_min_block_size) || in nvme_attach()
5129 (nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) { in nvme_attach()
5131 "using default %d", ISP2(nvme->n_min_block_size) ? in nvme_attach()
5134 nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE; in nvme_attach()
5137 if (nvme->n_submission_queues != -1 && in nvme_attach()
5138 (nvme->n_submission_queues < 1 || in nvme_attach()
5139 nvme->n_submission_queues > UINT16_MAX)) { in nvme_attach()
5141 "valid. Must be [1..%d]", nvme->n_submission_queues, in nvme_attach()
5143 nvme->n_submission_queues = -1; in nvme_attach()
5146 if (nvme->n_completion_queues != -1 && in nvme_attach()
5147 (nvme->n_completion_queues < 1 || in nvme_attach()
5148 nvme->n_completion_queues > UINT16_MAX)) { in nvme_attach()
5150 "valid. Must be [1..%d]", nvme->n_completion_queues, in nvme_attach()
5152 nvme->n_completion_queues = -1; in nvme_attach()
5155 if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) in nvme_attach()
5156 nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; in nvme_attach()
5157 else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) in nvme_attach()
5158 nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; in nvme_attach()
5160 if (nvme->n_io_squeue_len < NVME_MIN_IO_QUEUE_LEN) in nvme_attach()
5161 nvme->n_io_squeue_len = NVME_MIN_IO_QUEUE_LEN; in nvme_attach()
5162 if (nvme->n_io_cqueue_len < NVME_MIN_IO_QUEUE_LEN) in nvme_attach()
5163 nvme->n_io_cqueue_len = NVME_MIN_IO_QUEUE_LEN; in nvme_attach()
5165 if (nvme->n_async_event_limit < 1) in nvme_attach()
5166 nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; in nvme_attach()
5168 nvme->n_reg_acc_attr = nvme_reg_acc_attr; in nvme_attach()
5169 nvme->n_queue_dma_attr = nvme_queue_dma_attr; in nvme_attach()
5170 nvme->n_prp_dma_attr = nvme_prp_dma_attr; in nvme_attach()
5171 nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; in nvme_attach()
5176 nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, in nvme_attach()
5181 ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); in nvme_attach()
5183 if (nvme->n_fm_cap) { in nvme_attach()
5184 if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) in nvme_attach()
5185 nvme->n_reg_acc_attr.devacc_attr_access = in nvme_attach()
5188 if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { in nvme_attach()
5189 nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; in nvme_attach()
5190 nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; in nvme_attach()
5193 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || in nvme_attach()
5194 DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) in nvme_attach()
5197 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) in nvme_attach()
5199 (void *)nvme); in nvme_attach()
5202 nvme->n_progress |= NVME_FMA_INIT; in nvme_attach()
5213 if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, in nvme_attach()
5214 &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { in nvme_attach()
5219 nvme->n_progress |= NVME_REGS_MAPPED; in nvme_attach()
5224 if (!nvme_stat_init(nvme)) { in nvme_attach()
5228 nvme->n_progress |= NVME_STAT_INIT; in nvme_attach()
5235 nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), in nvme_attach()
5237 NULL, (void *)nvme, NULL, 0); in nvme_attach()
5239 if (nvme_init(nvme) != DDI_SUCCESS) in nvme_attach()
5246 &nvme->n_ufmh, nvme) != 0) { in nvme_attach()
5250 mutex_init(&nvme->n_fwslot_mutex, NULL, MUTEX_DRIVER, NULL); in nvme_attach()
5251 ddi_ufm_update(nvme->n_ufmh); in nvme_attach()
5252 nvme->n_progress |= NVME_UFM_INIT; in nvme_attach()
5254 nvme_mgmt_lock_init(&nvme->n_mgmt); in nvme_attach()
5255 nvme_lock_init(&nvme->n_lock); in nvme_attach()
5256 nvme->n_progress |= NVME_MGMT_INIT; in nvme_attach()
5261 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_attach()
5264 for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) { in nvme_attach()
5265 nvme_namespace_t *ns = nvme_nsid2ns(nvme, i); in nvme_attach()
5279 if (nvme_init_ns(nvme, i) != 0) { in nvme_attach()
5280 nvme_mgmt_unlock(nvme); in nvme_attach()
5297 NVME_MINOR_MAX, nvme->n_namespace_count); in nvme_attach()
5303 if (ddi_create_minor_node(nvme->n_dip, ns->ns_name, S_IFCHR, in nvme_attach()
5304 NVME_MINOR(ddi_get_instance(nvme->n_dip), i), in nvme_attach()
5306 nvme_mgmt_unlock(nvme); in nvme_attach()
5318 nvme->n_progress |= NVME_NS_INIT; in nvme_attach()
5323 nvme_mgmt_unlock(nvme); in nvme_attach()
5333 for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) { in nvme_attach()
5334 nvme_namespace_t *ns = nvme_nsid2ns(nvme, i); in nvme_attach()
5340 if (!nvme_bd_attach_ns(nvme, &com) && com.nioc_drv_err != in nvme_attach()
5342 dev_err(nvme->n_dip, CE_WARN, "!failed to attach " in nvme_attach()
5348 nvme_mgmt_unlock(nvme); in nvme_attach()
5367 nvme->n_async_event_supported = B_TRUE; in nvme_attach()
5368 for (uint16_t i = 0; i < nvme->n_async_event_limit; i++) { in nvme_attach()
5369 nvme_async_event(nvme); in nvme_attach()
5377 if (nvme->n_dead) in nvme_attach()
5389 nvme_t *nvme; in nvme_detach() local
5396 nvme = ddi_get_soft_state(nvme_state, instance); in nvme_detach()
5398 if (nvme == NULL) in nvme_detach()
5414 if (nvme->n_ev_rm_cb_id != NULL) { in nvme_detach()
5415 (void) ddi_remove_event_handler(nvme->n_ev_rm_cb_id); in nvme_detach()
5417 nvme->n_ev_rm_cb_id = NULL; in nvme_detach()
5429 if (nvme->n_ns) { in nvme_detach()
5430 for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) { in nvme_detach()
5431 nvme_namespace_t *ns = nvme_nsid2ns(nvme, i); in nvme_detach()
5448 kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * in nvme_detach()
5449 nvme->n_namespace_count); in nvme_detach()
5452 if (nvme->n_progress & NVME_MGMT_INIT) { in nvme_detach()
5453 nvme_lock_fini(&nvme->n_lock); in nvme_detach()
5454 nvme_mgmt_lock_fini(&nvme->n_mgmt); in nvme_detach()
5457 if (nvme->n_progress & NVME_UFM_INIT) { in nvme_detach()
5458 ddi_ufm_fini(nvme->n_ufmh); in nvme_detach()
5459 mutex_destroy(&nvme->n_fwslot_mutex); in nvme_detach()
5462 if (nvme->n_progress & NVME_INTERRUPTS) in nvme_detach()
5463 nvme_release_interrupts(nvme); in nvme_detach()
5465 for (uint_t i = 0; i < nvme->n_cq_count; i++) { in nvme_detach()
5466 if (nvme->n_cq[i]->ncq_cmd_taskq != NULL) in nvme_detach()
5467 taskq_wait(nvme->n_cq[i]->ncq_cmd_taskq); in nvme_detach()
5470 if (nvme->n_progress & NVME_MUTEX_INIT) { in nvme_detach()
5471 mutex_destroy(&nvme->n_minor_mutex); in nvme_detach()
5474 if (nvme->n_ioq_count > 0) { in nvme_detach()
5475 for (uint_t i = 1; i != nvme->n_ioq_count + 1; i++) { in nvme_detach()
5476 if (nvme->n_ioq[i] != NULL) { in nvme_detach()
5478 nvme_free_qpair(nvme->n_ioq[i]); in nvme_detach()
5482 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * in nvme_detach()
5483 (nvme->n_ioq_count + 1)); in nvme_detach()
5486 if (nvme->n_prp_cache != NULL) { in nvme_detach()
5487 kmem_cache_destroy(nvme->n_prp_cache); in nvme_detach()
5490 if (nvme->n_progress & NVME_REGS_MAPPED) { in nvme_detach()
5491 nvme_shutdown(nvme, B_FALSE); in nvme_detach()
5492 (void) nvme_reset(nvme, B_FALSE); in nvme_detach()
5495 if (nvme->n_progress & NVME_CTRL_LIMITS) in nvme_detach()
5496 sema_destroy(&nvme->n_abort_sema); in nvme_detach()
5498 if (nvme->n_progress & NVME_ADMIN_QUEUE) in nvme_detach()
5499 nvme_free_qpair(nvme->n_adminq); in nvme_detach()
5501 if (nvme->n_cq_count > 0) { in nvme_detach()
5502 nvme_destroy_cq_array(nvme, 0); in nvme_detach()
5503 nvme->n_cq = NULL; in nvme_detach()
5504 nvme->n_cq_count = 0; in nvme_detach()
5507 if (nvme->n_idcomns) in nvme_detach()
5508 kmem_free(nvme->n_idcomns, NVME_IDENTIFY_BUFSIZE); in nvme_detach()
5510 if (nvme->n_idctl) in nvme_detach()
5511 kmem_free(nvme->n_idctl, NVME_IDENTIFY_BUFSIZE); in nvme_detach()
5513 if (nvme->n_progress & NVME_REGS_MAPPED) in nvme_detach()
5514 ddi_regs_map_free(&nvme->n_regh); in nvme_detach()
5516 if (nvme->n_progress & NVME_STAT_INIT) in nvme_detach()
5517 nvme_stat_cleanup(nvme); in nvme_detach()
5519 if (nvme->n_progress & NVME_FMA_INIT) { in nvme_detach()
5520 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) in nvme_detach()
5521 ddi_fm_handler_unregister(nvme->n_dip); in nvme_detach()
5523 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || in nvme_detach()
5524 DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) in nvme_detach()
5525 pci_ereport_teardown(nvme->n_dip); in nvme_detach()
5527 ddi_fm_fini(nvme->n_dip); in nvme_detach()
5530 if (nvme->n_progress & NVME_PCI_CONFIG) in nvme_detach()
5531 pci_config_teardown(&nvme->n_pcicfg_handle); in nvme_detach()
5533 if (nvme->n_vendor != NULL) in nvme_detach()
5534 strfree(nvme->n_vendor); in nvme_detach()
5536 if (nvme->n_product != NULL) in nvme_detach()
5537 strfree(nvme->n_product); in nvme_detach()
5548 nvme_t *nvme; in nvme_quiesce() local
5552 nvme = ddi_get_soft_state(nvme_state, instance); in nvme_quiesce()
5554 if (nvme == NULL) in nvme_quiesce()
5557 nvme_shutdown(nvme, B_TRUE); in nvme_quiesce()
5559 (void) nvme_reset(nvme, B_TRUE); in nvme_quiesce()
5567 nvme_t *nvme = cmd->nc_nvme; in nvme_fill_prp() local
5597 nprp_per_page = nvme->n_pagesize / sizeof (uint64_t); in nvme_fill_prp()
5613 cmd->nc_prp = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); in nvme_fill_prp()
5645 nvme_t *nvme = cmd->nc_nvme; in nvme_fill_ranges() local
5663 cmd->nc_prp = kmem_cache_alloc(nvme->n_prp_cache, allocflag); in nvme_fill_ranges()
5696 nvme_t *nvme = ns->ns_nvme; in nvme_create_nvm_cmd() local
5704 cmd = nvme_alloc_cmd(nvme, allocflag); in nvme_create_nvm_cmd()
5768 nvme_t *nvme = ns->ns_nvme; in nvme_bd_driveinfo() local
5769 uint_t ns_count = MAX(1, nvme->n_namespaces_attachable); in nvme_bd_driveinfo()
5771 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_BDRO); in nvme_bd_driveinfo()
5778 drive->d_qcount = nvme->n_ioq_count; in nvme_bd_driveinfo()
5799 drive->d_qsize = nvme->n_io_squeue_len / ns_count; in nvme_bd_driveinfo()
5818 drive->d_model = nvme->n_idctl->id_model; in nvme_bd_driveinfo()
5819 drive->d_model_len = sizeof (nvme->n_idctl->id_model); in nvme_bd_driveinfo()
5820 drive->d_vendor = nvme->n_vendor; in nvme_bd_driveinfo()
5821 drive->d_vendor_len = strlen(nvme->n_vendor); in nvme_bd_driveinfo()
5822 drive->d_product = nvme->n_product; in nvme_bd_driveinfo()
5823 drive->d_product_len = strlen(nvme->n_product); in nvme_bd_driveinfo()
5824 drive->d_serial = nvme->n_idctl->id_serial; in nvme_bd_driveinfo()
5825 drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); in nvme_bd_driveinfo()
5826 drive->d_revision = nvme->n_idctl->id_fwrev; in nvme_bd_driveinfo()
5827 drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); in nvme_bd_driveinfo()
5834 if (nvme->n_idctl->id_oncs.on_dset_mgmt) in nvme_bd_driveinfo()
5837 nvme_mgmt_unlock(nvme); in nvme_bd_driveinfo()
5844 nvme_t *nvme = ns->ns_nvme; in nvme_bd_mediainfo() local
5846 if (nvme->n_dead) { in nvme_bd_mediainfo()
5850 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_BDRO); in nvme_bd_mediainfo()
5859 nvme_mgmt_unlock(nvme); in nvme_bd_mediainfo()
5867 nvme_t *nvme = ns->ns_nvme; in nvme_bd_cmd() local
5873 if (nvme->n_dead) { in nvme_bd_cmd()
5882 ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); in nvme_bd_cmd()
5883 ioq = nvme->n_ioq[cmd->nc_sqid]; in nvme_bd_cmd()
5901 cmd = nvme_retrieve_cmd(nvme, ioq); in nvme_bd_cmd()
5958 nvme_t *nvme = ns->ns_nvme; in nvme_bd_devid() local
5960 if (nvme->n_dead) { in nvme_bd_devid()
5997 nvme_t *nvme; in nvme_open() local
6009 nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(m)); in nvme_open()
6012 if (nvme == NULL) in nvme_open()
6015 if (nsid > MIN(nvme->n_namespace_count, NVME_MINOR_MAX)) in nvme_open()
6018 if (nvme->n_dead) in nvme_open()
6033 minor->nm_ctrl_lock.nli_nvme = nvme; in nvme_open()
6036 minor->nm_ns_lock.nli_nvme = nvme; in nvme_open()
6044 minor->nm_ctrl = nvme; in nvme_open()
6046 minor->nm_ns = nvme_nsid2ns(nvme, nsid); in nvme_open()
6063 mutex_enter(&nvme->n_minor_mutex); in nvme_open()
6078 mutex_exit(&nvme->n_minor_mutex); in nvme_open()
6088 mutex_exit(&nvme->n_minor_mutex); in nvme_open()
6099 nvme_t *nvme; in nvme_close() local
6118 nvme = minor->nm_ctrl; in nvme_close()
6119 mutex_enter(&nvme->n_minor_mutex); in nvme_close()
6133 mutex_exit(&nvme->n_minor_mutex); in nvme_close()
6188 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_excl_check() local
6202 ns = nvme_nsid2ns(nvme, ioc->nioc_nsid); in nvme_ioctl_excl_check()
6205 mutex_enter(&nvme->n_minor_mutex); in nvme_ioctl_excl_check()
6206 ctrl_is_excl = nvme->n_lock.nl_writer != NULL; in nvme_ioctl_excl_check()
6207 have_ctrl = nvme->n_lock.nl_writer == &minor->nm_ctrl_lock; in nvme_ioctl_excl_check()
6227 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_excl_check()
6368 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_ctrl_info() local
6401 if (!nvme_identify(nvme, B_TRUE, &id, &idbuf)) { in nvme_ioctl_ctrl_info()
6411 bcopy(nvme->n_idcomns, &info->nci_common_ns, in nvme_ioctl_ctrl_info()
6414 info->nci_vers = nvme->n_version; in nvme_ioctl_ctrl_info()
6421 cap.r = nvme_get64(nvme, NVME_REG_CAP); in nvme_ioctl_ctrl_info()
6425 info->nci_nintrs = (uint32_t)nvme->n_intr_cnt; in nvme_ioctl_ctrl_info()
6441 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_ns_info() local
6468 ns = nvme_nsid2ns(nvme, ns_info->nni_common.nioc_nsid); in nvme_ioctl_ns_info()
6476 if (!nvme_identify(nvme, B_TRUE, &id, &idbuf)) { in nvme_ioctl_ns_info()
6483 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_ns_info()
6492 nvme_mgmt_unlock(nvme); in nvme_ioctl_ns_info()
6498 nvme_mgmt_unlock(nvme); in nvme_ioctl_ns_info()
6515 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_identify() local
6557 if (!nvme_validate_identify(nvme, &id, ns_minor)) { in nvme_ioctl_identify()
6561 if (nvme_identify(nvme, B_TRUE, &id, &idctl)) { in nvme_ioctl_identify()
6616 nvme_ioc_cmd(nvme_t *nvme, nvme_ioctl_common_t *ioc, nvme_ioc_cmd_args_t *args) in nvme_ioc_cmd() argument
6621 cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP); in nvme_ioc_cmd()
6643 if (nvme_zalloc_dma(nvme, args->ica_data_len, in nvme_ioc_cmd()
6644 args->ica_dma_flags, &nvme->n_prp_dma_attr, &cmd->nc_dma) != in nvme_ioc_cmd()
6646 dev_err(nvme->n_dip, CE_WARN, in nvme_ioc_cmd()
6697 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_get_logpage() local
6748 if (!nvme_validate_logpage(nvme, &log)) { in nvme_ioctl_get_logpage()
6752 if (nvme_get_logpage(nvme, B_TRUE, &log, &buf)) { in nvme_ioctl_get_logpage()
6803 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_get_feature() local
6852 if (!nvme_validate_get_feature(nvme, &feat)) { in nvme_ioctl_get_feature()
6871 if (!nvme_ioc_cmd(nvme, &feat.nigf_common, &args)) { in nvme_ioctl_get_feature()
6912 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_format() local
6929 if (!nvme_validate_format(nvme, &ioc)) { in nvme_ioctl_format()
6939 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_format()
6941 if (!nvme_no_blkdev_attached(nvme, ioc.nif_common.nioc_nsid)) { in nvme_ioctl_format()
6942 nvme_mgmt_unlock(nvme); in nvme_ioctl_format()
6948 nvme_namespace_t *ns = nvme_nsid2ns(nvme, in nvme_ioctl_format()
6953 nvme_mgmt_unlock(nvme); in nvme_ioctl_format()
6958 if (nvme_format_nvm(nvme, &ioc)) { in nvme_ioctl_format()
6960 nvme_rescan_ns(nvme, ioc.nif_common.nioc_nsid); in nvme_ioctl_format()
6962 nvme_mgmt_unlock(nvme); in nvme_ioctl_format()
6977 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_bd_detach() local
6995 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_bd_detach()
6996 if (nvme_bd_detach_ns(nvme, &com)) { in nvme_ioctl_bd_detach()
6999 nvme_mgmt_unlock(nvme); in nvme_ioctl_bd_detach()
7014 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_bd_attach() local
7033 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_bd_attach()
7034 ns = nvme_nsid2ns(nvme, com.nioc_nsid); in nvme_ioctl_bd_attach()
7044 nvme_rescan_ns(nvme, com.nioc_nsid); in nvme_ioctl_bd_attach()
7047 if (nvme_bd_attach_ns(nvme, &com)) { in nvme_ioctl_bd_attach()
7050 nvme_mgmt_unlock(nvme); in nvme_ioctl_bd_attach()
7069 nvme_ctrl_attach_detach_ns(nvme_t *nvme, nvme_namespace_t *ns, in nvme_ctrl_attach_detach_ns() argument
7077 ASSERT(nvme_mgmt_lock_held(nvme)); in nvme_ctrl_attach_detach_ns()
7094 ctrlids[1] = nvme->n_idctl->id_cntlid; in nvme_ctrl_attach_detach_ns()
7103 return (nvme_ioc_cmd(nvme, ioc, &args)); in nvme_ctrl_attach_detach_ns()
7110 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_ctrl_detach() local
7129 if (!nvme_validate_ctrl_attach_detach_ns(nvme, &com)) { in nvme_ioctl_ctrl_detach()
7133 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_ctrl_detach()
7134 ns = nvme_nsid2ns(nvme, com.nioc_nsid); in nvme_ioctl_ctrl_detach()
7137 if (nvme_ctrl_attach_detach_ns(nvme, ns, &com, B_FALSE)) { in nvme_ioctl_ctrl_detach()
7138 nvme_rescan_ns(nvme, com.nioc_nsid); in nvme_ioctl_ctrl_detach()
7142 nvme_mgmt_unlock(nvme); in nvme_ioctl_ctrl_detach()
7157 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_ns_create() local
7176 if (!nvme_validate_ns_create(nvme, &create)) { in nvme_ioctl_ns_create()
7227 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_ns_create()
7228 if (nvme_ioc_cmd(nvme, &create.nnc_common, &args)) { in nvme_ioctl_ns_create()
7230 nvme_rescan_ns(nvme, create.nnc_nsid); in nvme_ioctl_ns_create()
7233 nvme_mgmt_unlock(nvme); in nvme_ioctl_ns_create()
7250 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_ns_delete() local
7268 if (!nvme_validate_ns_delete(nvme, &com)) { in nvme_ioctl_ns_delete()
7272 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_ns_delete()
7274 if (!nvme_no_blkdev_attached(nvme, com.nioc_nsid)) { in nvme_ioctl_ns_delete()
7275 nvme_mgmt_unlock(nvme); in nvme_ioctl_ns_delete()
7281 nvme_namespace_t *ns = nvme_nsid2ns(nvme, com.nioc_nsid); in nvme_ioctl_ns_delete()
7284 nvme_mgmt_unlock(nvme); in nvme_ioctl_ns_delete()
7308 if (nvme_ioc_cmd(nvme, &com, &args)) { in nvme_ioctl_ns_delete()
7309 nvme_rescan_ns(nvme, com.nioc_nsid); in nvme_ioctl_ns_delete()
7312 nvme_mgmt_unlock(nvme); in nvme_ioctl_ns_delete()
7327 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_ctrl_attach() local
7346 if (!nvme_validate_ctrl_attach_detach_ns(nvme, &com)) { in nvme_ioctl_ctrl_attach()
7350 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_ctrl_attach()
7351 ns = nvme_nsid2ns(nvme, com.nioc_nsid); in nvme_ioctl_ctrl_attach()
7354 if (nvme_ctrl_attach_detach_ns(nvme, ns, &com, B_TRUE)) { in nvme_ioctl_ctrl_attach()
7355 nvme_rescan_ns(nvme, com.nioc_nsid); in nvme_ioctl_ctrl_attach()
7359 nvme_mgmt_unlock(nvme); in nvme_ioctl_ctrl_attach()
7371 nvme_ufm_update(nvme_t *nvme) in nvme_ufm_update() argument
7373 mutex_enter(&nvme->n_fwslot_mutex); in nvme_ufm_update()
7374 ddi_ufm_update(nvme->n_ufmh); in nvme_ufm_update()
7375 if (nvme->n_fwslot != NULL) { in nvme_ufm_update()
7376 kmem_free(nvme->n_fwslot, sizeof (nvme_fwslot_log_t)); in nvme_ufm_update()
7377 nvme->n_fwslot = NULL; in nvme_ufm_update()
7379 mutex_exit(&nvme->n_fwslot_mutex); in nvme_ufm_update()
7393 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_firmware_download() local
7419 if (!nvme_validate_fw_load(nvme, &fw)) { in nvme_ioctl_firmware_download()
7440 data.vcd_vers = &nvme->n_version; in nvme_ioctl_firmware_download()
7441 data.vcd_id = nvme->n_idctl; in nvme_ioctl_firmware_download()
7444 if ((nvme->n_max_data_transfer_size % gran) == 0) { in nvme_ioctl_firmware_download()
7445 maxcopy = nvme->n_max_data_transfer_size; in nvme_ioctl_firmware_download()
7446 } else if (gran <= nvme->n_max_data_transfer_size) { in nvme_ioctl_firmware_download()
7468 if (!nvme_ioc_cmd(nvme, &fw.fwl_common, &args)) { in nvme_ioctl_firmware_download()
7490 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_firmware_commit() local
7513 if (!nvme_validate_fw_commit(nvme, &fw)) { in nvme_ioctl_firmware_commit()
7529 (void) nvme_ioc_cmd(nvme, &fw.fwc_common, &args); in nvme_ioctl_firmware_commit()
7536 nvme_ufm_update(nvme); in nvme_ioctl_firmware_commit()
7640 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_passthru() local
7664 if (!nvme_validate_vuc(nvme, &pass)) { in nvme_ioctl_passthru()
7668 nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME); in nvme_ioctl_passthru()
7675 if (!nvme_no_blkdev_attached(nvme, NVME_NSID_BCAST)) { in nvme_ioctl_passthru()
7676 nvme_mgmt_unlock(nvme); in nvme_ioctl_passthru()
7704 if (nvme_ioc_cmd(nvme, &pass.npc_common, &args)) { in nvme_ioctl_passthru()
7707 nvme_rescan_ns(nvme, NVME_NSID_BCAST); in nvme_ioctl_passthru()
7710 nvme_mgmt_unlock(nvme); in nvme_ioctl_passthru()
7725 nvme_t *nvme = minor->nm_ctrl; in nvme_ioctl_lock() local
7798 mutex_enter(&nvme->n_minor_mutex); in nvme_ioctl_lock()
7803 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_lock()
7814 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_lock()
7822 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_lock()
7831 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_lock()
7839 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_lock()
7882 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_lock()
7898 nvme_t *const nvme = minor->nm_ctrl; in nvme_ioctl_unlock() local
7939 mutex_enter(&nvme->n_minor_mutex); in nvme_ioctl_unlock()
7942 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_unlock()
7949 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_unlock()
7963 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_unlock()
7971 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_unlock()
7983 lock = &nvme->n_lock; in nvme_ioctl_unlock()
7989 ns = nvme_nsid2ns(nvme, nsid); in nvme_ioctl_unlock()
7995 mutex_exit(&nvme->n_minor_mutex); in nvme_ioctl_unlock()
8016 nvme_t *nvme; in nvme_ioctl() local
8023 nvme = minor->nm_ctrl; in nvme_ioctl()
8024 if (nvme == NULL) in nvme_ioctl()
8028 return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0)); in nvme_ioctl()
8030 if (nvme->n_dead && (cmd != NVME_IOC_BD_DETACH && cmd != in nvme_ioctl()
8036 return (nvme_ioctl_copyout_error(nvme->n_dead_status, arg, in nvme_ioctl()
8100 ASSERT(!nvme_mgmt_lock_held(nvme)); in nvme_ioctl()
8111 nvme_t *nvme = arg; in nvme_ufm_fill_image() local
8117 ddi_ufm_image_set_nslots(img, nvme->n_idctl->id_frmw.fw_nslot); in nvme_ufm_fill_image()
8136 nvme_t *nvme = arg; in nvme_ufm_fill_slot() local
8142 if (imgno > 0 || slotno > (nvme->n_idctl->id_frmw.fw_nslot - 1)) in nvme_ufm_fill_slot()
8145 mutex_enter(&nvme->n_fwslot_mutex); in nvme_ufm_fill_slot()
8146 if (nvme->n_fwslot == NULL) { in nvme_ufm_fill_slot()
8147 if (!nvme_get_logpage_int(nvme, B_TRUE, &log, &bufsize, in nvme_ufm_fill_slot()
8152 mutex_exit(&nvme->n_fwslot_mutex); in nvme_ufm_fill_slot()
8155 nvme->n_fwslot = (nvme_fwslot_log_t *)log; in nvme_ufm_fill_slot()
8161 if (slotno == (nvme->n_fwslot->fw_afi - 1)) in nvme_ufm_fill_slot()
8164 if (slotno != 0 || nvme->n_idctl->id_frmw.fw_readonly == 0) in nvme_ufm_fill_slot()
8167 if (nvme->n_fwslot->fw_frs[slotno][0] == '\0') { in nvme_ufm_fill_slot()
8170 (void) strncpy(fw_ver, nvme->n_fwslot->fw_frs[slotno], in nvme_ufm_fill_slot()
8175 mutex_exit(&nvme->n_fwslot_mutex); in nvme_ufm_fill_slot()
8190 nvme_ctrl_atleast(nvme_t *nvme, const nvme_version_t *min) in nvme_ctrl_atleast() argument
8192 return (nvme_vers_atleast(&nvme->n_version, min) ? B_TRUE : B_FALSE); in nvme_ctrl_atleast()