Lines Matching refs:cmd
855 nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); in nvme_alloc_cmd() local
857 if (cmd == NULL) in nvme_alloc_cmd()
858 return (cmd); in nvme_alloc_cmd()
860 bzero(cmd, sizeof (nvme_cmd_t)); in nvme_alloc_cmd()
862 cmd->nc_nvme = nvme; in nvme_alloc_cmd()
864 mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, in nvme_alloc_cmd()
866 cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); in nvme_alloc_cmd()
868 return (cmd); in nvme_alloc_cmd()
872 nvme_free_cmd(nvme_cmd_t *cmd) in nvme_free_cmd() argument
875 if (list_link_active(&cmd->nc_list)) in nvme_free_cmd()
878 if (cmd->nc_dma) { in nvme_free_cmd()
879 if (cmd->nc_dma->nd_cached) in nvme_free_cmd()
880 kmem_cache_free(cmd->nc_nvme->n_prp_cache, in nvme_free_cmd()
881 cmd->nc_dma); in nvme_free_cmd()
883 nvme_free_dma(cmd->nc_dma); in nvme_free_cmd()
884 cmd->nc_dma = NULL; in nvme_free_cmd()
887 cv_destroy(&cmd->nc_cv); in nvme_free_cmd()
888 mutex_destroy(&cmd->nc_mutex); in nvme_free_cmd()
890 kmem_cache_free(nvme_cmd_cache, cmd); in nvme_free_cmd()
894 nvme_submit_admin_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) in nvme_submit_admin_cmd() argument
897 nvme_submit_cmd_common(qp, cmd); in nvme_submit_admin_cmd()
901 nvme_submit_io_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) in nvme_submit_io_cmd() argument
906 nvme_submit_cmd_common(qp, cmd); in nvme_submit_io_cmd()
911 nvme_submit_cmd_common(nvme_qpair_t *qp, nvme_cmd_t *cmd) in nvme_submit_cmd_common() argument
916 cmd->nc_completed = B_FALSE; in nvme_submit_cmd_common()
926 qp->nq_cmd[qp->nq_next_cmd] = cmd; in nvme_submit_cmd_common()
930 cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; in nvme_submit_cmd_common()
931 bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); in nvme_submit_cmd_common()
938 nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); in nvme_submit_cmd_common()
946 nvme_cmd_t *cmd; in nvme_unqueue_cmd() local
951 cmd = qp->nq_cmd[cid]; in nvme_unqueue_cmd()
957 ASSERT3P(cmd, !=, NULL); in nvme_unqueue_cmd()
958 ASSERT3P(cmd->nc_nvme, ==, nvme); in nvme_unqueue_cmd()
959 ASSERT3S(cmd->nc_sqe.sqe_cid, ==, cid); in nvme_unqueue_cmd()
961 return (cmd); in nvme_unqueue_cmd()
970 nvme_cmd_t *cmd; in nvme_retrieve_cmd() local
986 cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid); in nvme_retrieve_cmd()
988 ASSERT(cmd->nc_sqid == cqe->cqe_sqid); in nvme_retrieve_cmd()
989 bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); in nvme_retrieve_cmd()
999 nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); in nvme_retrieve_cmd()
1002 return (cmd); in nvme_retrieve_cmd()
1006 nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) in nvme_check_unknown_cmd_status() argument
1008 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_unknown_cmd_status()
1010 dev_err(cmd->nc_nvme->n_dip, CE_WARN, in nvme_check_unknown_cmd_status()
1012 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, in nvme_check_unknown_cmd_status()
1016 if (cmd->nc_xfer != NULL) in nvme_check_unknown_cmd_status()
1017 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_unknown_cmd_status()
1019 if (cmd->nc_nvme->n_strict_version) { in nvme_check_unknown_cmd_status()
1020 cmd->nc_nvme->n_dead = B_TRUE; in nvme_check_unknown_cmd_status()
1021 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); in nvme_check_unknown_cmd_status()
1028 nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) in nvme_check_vendor_cmd_status() argument
1030 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_vendor_cmd_status()
1032 dev_err(cmd->nc_nvme->n_dip, CE_WARN, in nvme_check_vendor_cmd_status()
1034 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, in nvme_check_vendor_cmd_status()
1037 if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { in nvme_check_vendor_cmd_status()
1038 cmd->nc_nvme->n_dead = B_TRUE; in nvme_check_vendor_cmd_status()
1039 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); in nvme_check_vendor_cmd_status()
1046 nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) in nvme_check_integrity_cmd_status() argument
1048 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_integrity_cmd_status()
1054 if (cmd->nc_xfer != NULL) in nvme_check_integrity_cmd_status()
1055 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); in nvme_check_integrity_cmd_status()
1061 if (cmd->nc_xfer != NULL) in nvme_check_integrity_cmd_status()
1062 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); in nvme_check_integrity_cmd_status()
1066 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_integrity_cmd_status()
1071 nvme_check_generic_cmd_status(nvme_cmd_t *cmd) in nvme_check_generic_cmd_status() argument
1073 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_generic_cmd_status()
1084 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_generic_cmd_status()
1085 "invalid opcode in cmd %p", (void *)cmd); in nvme_check_generic_cmd_status()
1090 if (!cmd->nc_dontpanic) in nvme_check_generic_cmd_status()
1091 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, in nvme_check_generic_cmd_status()
1093 (void *)cmd); in nvme_check_generic_cmd_status()
1098 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_generic_cmd_status()
1099 "cmd ID conflict in cmd %p", (void *)cmd); in nvme_check_generic_cmd_status()
1104 if (!cmd->nc_dontpanic) in nvme_check_generic_cmd_status()
1105 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, in nvme_check_generic_cmd_status()
1107 (void *)cmd); in nvme_check_generic_cmd_status()
1112 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_generic_cmd_status()
1113 "LBA out of range in cmd %p", (void *)cmd); in nvme_check_generic_cmd_status()
1122 atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); in nvme_check_generic_cmd_status()
1123 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1124 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); in nvme_check_generic_cmd_status()
1134 atomic_inc_32(&cmd->nc_nvme->n_internal_err); in nvme_check_generic_cmd_status()
1135 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1136 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); in nvme_check_generic_cmd_status()
1145 atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); in nvme_check_generic_cmd_status()
1150 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); in nvme_check_generic_cmd_status()
1151 cmd->nc_nvme->n_dead = B_TRUE; in nvme_check_generic_cmd_status()
1156 atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); in nvme_check_generic_cmd_status()
1161 atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); in nvme_check_generic_cmd_status()
1162 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1163 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); in nvme_check_generic_cmd_status()
1168 atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); in nvme_check_generic_cmd_status()
1169 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1170 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); in nvme_check_generic_cmd_status()
1174 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_generic_cmd_status()
1179 nvme_check_specific_cmd_status(nvme_cmd_t *cmd) in nvme_check_specific_cmd_status() argument
1181 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_specific_cmd_status()
1186 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); in nvme_check_specific_cmd_status()
1187 atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); in nvme_check_specific_cmd_status()
1192 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || in nvme_check_specific_cmd_status()
1193 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || in nvme_check_specific_cmd_status()
1194 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || in nvme_check_specific_cmd_status()
1195 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); in nvme_check_specific_cmd_status()
1196 atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); in nvme_check_specific_cmd_status()
1201 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || in nvme_check_specific_cmd_status()
1202 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); in nvme_check_specific_cmd_status()
1203 atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); in nvme_check_specific_cmd_status()
1208 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); in nvme_check_specific_cmd_status()
1209 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_specific_cmd_status()
1210 "abort command limit exceeded in cmd %p", (void *)cmd); in nvme_check_specific_cmd_status()
1215 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); in nvme_check_specific_cmd_status()
1216 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_specific_cmd_status()
1218 (void *)cmd); in nvme_check_specific_cmd_status()
1223 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); in nvme_check_specific_cmd_status()
1224 atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); in nvme_check_specific_cmd_status()
1229 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); in nvme_check_specific_cmd_status()
1230 atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); in nvme_check_specific_cmd_status()
1235 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); in nvme_check_specific_cmd_status()
1236 atomic_inc_32(&cmd->nc_nvme->n_inv_format); in nvme_check_specific_cmd_status()
1237 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
1238 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
1243 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); in nvme_check_specific_cmd_status()
1244 atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); in nvme_check_specific_cmd_status()
1249 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || in nvme_check_specific_cmd_status()
1250 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || in nvme_check_specific_cmd_status()
1251 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); in nvme_check_specific_cmd_status()
1252 atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); in nvme_check_specific_cmd_status()
1253 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
1254 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
1259 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || in nvme_check_specific_cmd_status()
1260 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || in nvme_check_specific_cmd_status()
1261 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); in nvme_check_specific_cmd_status()
1262 atomic_inc_32(&cmd->nc_nvme->n_inv_prot); in nvme_check_specific_cmd_status()
1263 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
1264 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
1269 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); in nvme_check_specific_cmd_status()
1270 atomic_inc_32(&cmd->nc_nvme->n_readonly); in nvme_check_specific_cmd_status()
1271 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
1272 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
1276 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_specific_cmd_status()
1281 nvme_check_cmd_status(nvme_cmd_t *cmd) in nvme_check_cmd_status() argument
1283 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_cmd_status()
1289 if (cmd->nc_nvme->n_dead) in nvme_check_cmd_status()
1297 return (nvme_check_generic_cmd_status(cmd)); in nvme_check_cmd_status()
1299 return (nvme_check_specific_cmd_status(cmd)); in nvme_check_cmd_status()
1301 return (nvme_check_integrity_cmd_status(cmd)); in nvme_check_cmd_status()
1303 return (nvme_check_vendor_cmd_status(cmd)); in nvme_check_cmd_status()
1305 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_cmd_status()
1312 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_abort_cmd() local
1321 cmd->nc_sqid = 0; in nvme_abort_cmd()
1322 cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; in nvme_abort_cmd()
1323 cmd->nc_callback = nvme_wakeup_cmd; in nvme_abort_cmd()
1324 cmd->nc_sqe.sqe_cdw10 = ac.r; in nvme_abort_cmd()
1332 nvme_admin_cmd(cmd, sec); in nvme_abort_cmd()
1335 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_abort_cmd()
1338 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_abort_cmd()
1344 cmd->nc_cqe.cqe_dw0 & 1 ? "un" : ""); in nvme_abort_cmd()
1345 if ((cmd->nc_cqe.cqe_dw0 & 1) == 0) in nvme_abort_cmd()
1349 nvme_free_cmd(cmd); in nvme_abort_cmd()
1360 nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) in nvme_wait_cmd() argument
1363 nvme_t *nvme = cmd->nc_nvme; in nvme_wait_cmd()
1367 ASSERT(mutex_owned(&cmd->nc_mutex)); in nvme_wait_cmd()
1369 while (!cmd->nc_completed) { in nvme_wait_cmd()
1370 if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) in nvme_wait_cmd()
1374 if (cmd->nc_completed) in nvme_wait_cmd()
1386 "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_cid, cmd->nc_sqid, in nvme_wait_cmd()
1387 cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); in nvme_wait_cmd()
1392 nvme_check_dma_hdl(cmd->nc_dma) || in nvme_wait_cmd()
1393 cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { in nvme_wait_cmd()
1396 } else if (nvme_abort_cmd(cmd, sec) == 0) { in nvme_wait_cmd()
1401 while (!cmd->nc_completed) in nvme_wait_cmd()
1402 cv_wait(&cmd->nc_cv, &cmd->nc_mutex); in nvme_wait_cmd()
1407 qp = nvme->n_ioq[cmd->nc_sqid]; in nvme_wait_cmd()
1410 (void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); in nvme_wait_cmd()
1418 if (cmd->nc_dma != NULL) { in nvme_wait_cmd()
1420 list_insert_head(&nvme_lost_cmds, cmd); in nvme_wait_cmd()
1428 nvme_cmd_t *cmd = arg; in nvme_wakeup_cmd() local
1430 mutex_enter(&cmd->nc_mutex); in nvme_wakeup_cmd()
1431 cmd->nc_completed = B_TRUE; in nvme_wakeup_cmd()
1432 cv_signal(&cmd->nc_cv); in nvme_wakeup_cmd()
1433 mutex_exit(&cmd->nc_mutex); in nvme_wakeup_cmd()
1439 nvme_cmd_t *cmd = arg; in nvme_async_event_task() local
1440 nvme_t *nvme = cmd->nc_nvme; in nvme_async_event_task()
1456 if (nvme_check_cmd_status(cmd) != 0) { in nvme_async_event_task()
1457 dev_err(cmd->nc_nvme->n_dip, CE_WARN, in nvme_async_event_task()
1459 "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, in nvme_async_event_task()
1460 cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, in nvme_async_event_task()
1461 cmd->nc_cqe.cqe_sf.sf_m); in nvme_async_event_task()
1463 if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && in nvme_async_event_task()
1464 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { in nvme_async_event_task()
1465 cmd->nc_nvme->n_dead = B_TRUE; in nvme_async_event_task()
1466 ddi_fm_service_impact(cmd->nc_nvme->n_dip, in nvme_async_event_task()
1469 nvme_free_cmd(cmd); in nvme_async_event_task()
1474 event.r = cmd->nc_cqe.cqe_dw0; in nvme_async_event_task()
1477 bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); in nvme_async_event_task()
1478 nvme_submit_admin_cmd(nvme->n_adminq, cmd); in nvme_async_event_task()
1589 nvme_admin_cmd(nvme_cmd_t *cmd, int sec) in nvme_admin_cmd() argument
1591 mutex_enter(&cmd->nc_mutex); in nvme_admin_cmd()
1592 nvme_submit_admin_cmd(cmd->nc_nvme->n_adminq, cmd); in nvme_admin_cmd()
1593 nvme_wait_cmd(cmd, sec); in nvme_admin_cmd()
1594 mutex_exit(&cmd->nc_mutex); in nvme_admin_cmd()
1600 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_async_event() local
1602 cmd->nc_sqid = 0; in nvme_async_event()
1603 cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; in nvme_async_event()
1604 cmd->nc_callback = nvme_async_event_task; in nvme_async_event()
1606 nvme_submit_admin_cmd(nvme->n_adminq, cmd); in nvme_async_event()
1613 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_format_nvm() local
1623 cmd->nc_sqid = 0; in nvme_format_nvm()
1624 cmd->nc_callback = nvme_wakeup_cmd; in nvme_format_nvm()
1625 cmd->nc_sqe.sqe_nsid = nsid; in nvme_format_nvm()
1626 cmd->nc_sqe.sqe_opc = NVME_OPC_NVM_FORMAT; in nvme_format_nvm()
1627 cmd->nc_sqe.sqe_cdw10 = format_nvm.r; in nvme_format_nvm()
1634 cmd->nc_dontpanic = B_TRUE; in nvme_format_nvm()
1636 nvme_admin_cmd(cmd, nvme_format_cmd_timeout); in nvme_format_nvm()
1638 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_format_nvm()
1641 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_format_nvm()
1644 nvme_free_cmd(cmd); in nvme_format_nvm()
1652 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_get_logpage() local
1659 cmd->nc_sqid = 0; in nvme_get_logpage()
1660 cmd->nc_callback = nvme_wakeup_cmd; in nvme_get_logpage()
1661 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; in nvme_get_logpage()
1667 cmd->nc_sqe.sqe_nsid = (uint32_t)-1; in nvme_get_logpage()
1677 cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); in nvme_get_logpage()
1682 cmd->nc_sqe.sqe_nsid = (uint32_t)-1; in nvme_get_logpage()
1698 cmd->nc_sqe.sqe_cdw10 = getlogpage.r; in nvme_get_logpage()
1701 DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { in nvme_get_logpage()
1708 if (cmd->nc_dma->nd_ncookie > 2) { in nvme_get_logpage()
1716 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_get_logpage()
1717 if (cmd->nc_dma->nd_ncookie > 1) { in nvme_get_logpage()
1718 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, in nvme_get_logpage()
1719 &cmd->nc_dma->nd_cookie); in nvme_get_logpage()
1720 cmd->nc_sqe.sqe_dptr.d_prp[1] = in nvme_get_logpage()
1721 cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_get_logpage()
1724 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_get_logpage()
1726 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_get_logpage()
1729 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_get_logpage()
1734 bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); in nvme_get_logpage()
1737 nvme_free_cmd(cmd); in nvme_get_logpage()
1745 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_identify() local
1751 cmd->nc_sqid = 0; in nvme_identify()
1752 cmd->nc_callback = nvme_wakeup_cmd; in nvme_identify()
1753 cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; in nvme_identify()
1754 cmd->nc_sqe.sqe_nsid = nsid; in nvme_identify()
1755 cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; in nvme_identify()
1758 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { in nvme_identify()
1765 if (cmd->nc_dma->nd_ncookie > 2) { in nvme_identify()
1773 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_identify()
1774 if (cmd->nc_dma->nd_ncookie > 1) { in nvme_identify()
1775 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, in nvme_identify()
1776 &cmd->nc_dma->nd_cookie); in nvme_identify()
1777 cmd->nc_sqe.sqe_dptr.d_prp[1] = in nvme_identify()
1778 cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_identify()
1781 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_identify()
1783 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_identify()
1786 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_identify()
1791 bcopy(cmd->nc_dma->nd_memp, *buf, NVME_IDENTIFY_BUFSIZE); in nvme_identify()
1794 nvme_free_cmd(cmd); in nvme_identify()
1804 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_set_features() local
1809 cmd->nc_sqid = 0; in nvme_set_features()
1810 cmd->nc_callback = nvme_wakeup_cmd; in nvme_set_features()
1811 cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; in nvme_set_features()
1812 cmd->nc_sqe.sqe_cdw10 = feature; in nvme_set_features()
1813 cmd->nc_sqe.sqe_cdw11 = val; in nvme_set_features()
1828 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_set_features()
1830 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_set_features()
1833 feature, cmd->nc_cqe.cqe_sf.sf_sct, in nvme_set_features()
1834 cmd->nc_cqe.cqe_sf.sf_sc); in nvme_set_features()
1838 *res = cmd->nc_cqe.cqe_dw0; in nvme_set_features()
1841 nvme_free_cmd(cmd); in nvme_set_features()
1849 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_get_features() local
1857 cmd->nc_sqid = 0; in nvme_get_features()
1858 cmd->nc_callback = nvme_wakeup_cmd; in nvme_get_features()
1859 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_FEATURES; in nvme_get_features()
1860 cmd->nc_sqe.sqe_cdw10 = feature; in nvme_get_features()
1861 cmd->nc_sqe.sqe_cdw11 = *res; in nvme_get_features()
1892 cmd->nc_dontpanic = B_TRUE; in nvme_get_features()
1893 cmd->nc_sqe.sqe_nsid = nsid; in nvme_get_features()
1913 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { in nvme_get_features()
1920 if (cmd->nc_dma->nd_ncookie > 2) { in nvme_get_features()
1928 cmd->nc_sqe.sqe_dptr.d_prp[0] = in nvme_get_features()
1929 cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_get_features()
1930 if (cmd->nc_dma->nd_ncookie > 1) { in nvme_get_features()
1931 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, in nvme_get_features()
1932 &cmd->nc_dma->nd_cookie); in nvme_get_features()
1933 cmd->nc_sqe.sqe_dptr.d_prp[1] = in nvme_get_features()
1934 cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_get_features()
1938 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_get_features()
1940 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_get_features()
1942 cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && in nvme_get_features()
1943 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_FLD) in nvme_get_features()
1948 feature, cmd->nc_cqe.cqe_sf.sf_sct, in nvme_get_features()
1949 cmd->nc_cqe.cqe_sf.sf_sc); in nvme_get_features()
1956 bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); in nvme_get_features()
1959 *res = cmd->nc_cqe.cqe_dw0; in nvme_get_features()
1962 nvme_free_cmd(cmd); in nvme_get_features()
2003 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_create_io_qpair() local
2016 cmd->nc_sqid = 0; in nvme_create_io_qpair()
2017 cmd->nc_callback = nvme_wakeup_cmd; in nvme_create_io_qpair()
2018 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; in nvme_create_io_qpair()
2019 cmd->nc_sqe.sqe_cdw10 = dw10.r; in nvme_create_io_qpair()
2020 cmd->nc_sqe.sqe_cdw11 = c_dw11.r; in nvme_create_io_qpair()
2021 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; in nvme_create_io_qpair()
2023 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_create_io_qpair()
2025 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_create_io_qpair()
2028 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_create_io_qpair()
2032 nvme_free_cmd(cmd); in nvme_create_io_qpair()
2037 cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_create_io_qpair()
2038 cmd->nc_sqid = 0; in nvme_create_io_qpair()
2039 cmd->nc_callback = nvme_wakeup_cmd; in nvme_create_io_qpair()
2040 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; in nvme_create_io_qpair()
2041 cmd->nc_sqe.sqe_cdw10 = dw10.r; in nvme_create_io_qpair()
2042 cmd->nc_sqe.sqe_cdw11 = s_dw11.r; in nvme_create_io_qpair()
2043 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; in nvme_create_io_qpair()
2045 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_create_io_qpair()
2047 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_create_io_qpair()
2050 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_create_io_qpair()
2055 nvme_free_cmd(cmd); in nvme_create_io_qpair()
2645 nvme_cmd_t *cmd; in nvme_intr() local
2662 while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { in nvme_intr()
2663 taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, in nvme_intr()
2664 cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); in nvme_intr()
2807 nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) in nvme_attach() argument
2816 if (cmd != DDI_ATTACH) in nvme_attach()
3000 nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) in nvme_detach() argument
3005 if (cmd != DDI_DETACH) in nvme_detach()
3125 nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) in nvme_fill_prp() argument
3127 nvme_t *nvme = cmd->nc_nvme; in nvme_fill_prp()
3134 cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; in nvme_fill_prp()
3138 cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; in nvme_fill_prp()
3141 cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; in nvme_fill_prp()
3158 cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); in nvme_fill_prp()
3159 bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); in nvme_fill_prp()
3161 cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_fill_prp()
3164 for (prp = (uint64_t *)cmd->nc_dma->nd_memp; in nvme_fill_prp()
3171 (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, in nvme_fill_prp()
3180 nvme_cmd_t *cmd; in nvme_create_nvm_cmd() local
3185 cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? in nvme_create_nvm_cmd()
3188 if (cmd == NULL) in nvme_create_nvm_cmd()
3191 cmd->nc_sqe.sqe_opc = opc; in nvme_create_nvm_cmd()
3192 cmd->nc_callback = nvme_bd_xfer_done; in nvme_create_nvm_cmd()
3193 cmd->nc_xfer = xfer; in nvme_create_nvm_cmd()
3200 cmd->nc_sqe.sqe_nsid = ns->ns_id; in nvme_create_nvm_cmd()
3202 cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; in nvme_create_nvm_cmd()
3203 cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); in nvme_create_nvm_cmd()
3204 cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); in nvme_create_nvm_cmd()
3206 if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) in nvme_create_nvm_cmd()
3211 cmd->nc_sqe.sqe_nsid = ns->ns_id; in nvme_create_nvm_cmd()
3218 return (cmd); in nvme_create_nvm_cmd()
3221 nvme_free_cmd(cmd); in nvme_create_nvm_cmd()
3228 nvme_cmd_t *cmd = arg; in nvme_bd_xfer_done() local
3229 bd_xfer_t *xfer = cmd->nc_xfer; in nvme_bd_xfer_done()
3232 error = nvme_check_cmd_status(cmd); in nvme_bd_xfer_done()
3233 nvme_free_cmd(cmd); in nvme_bd_xfer_done()
3296 nvme_cmd_t *cmd; in nvme_bd_cmd() local
3304 cmd = nvme_create_nvm_cmd(ns, opc, xfer); in nvme_bd_cmd()
3305 if (cmd == NULL) in nvme_bd_cmd()
3308 cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; in nvme_bd_cmd()
3309 ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); in nvme_bd_cmd()
3310 ioq = nvme->n_ioq[cmd->nc_sqid]; in nvme_bd_cmd()
3319 ret = nvme_submit_io_cmd(ioq, cmd); in nvme_bd_cmd()
3328 cmd = nvme_retrieve_cmd(nvme, ioq); in nvme_bd_cmd()
3329 if (cmd != NULL) in nvme_bd_cmd()
3330 nvme_bd_xfer_done(cmd); in nvme_bd_cmd()
3820 nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p, in nvme_ioctl() argument
3852 if (IS_DEVCTL(cmd)) in nvme_ioctl()
3853 return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0)); in nvme_ioctl()
3877 if (nvme->n_dead && cmd != NVME_IOC_DETACH) in nvme_ioctl()
3881 if (cmd == NVME_IOC_IDENTIFY_CTRL) { in nvme_ioctl()
3887 } else if (cmd == NVME_IOC_IDENTIFY_NSID && nsid == 0) { in nvme_ioctl()
3895 if (IS_NVME_IOC(cmd) && nvme_ioctl[NVME_IOC_CMD(cmd)] != NULL) in nvme_ioctl()
3896 rv = nvme_ioctl[NVME_IOC_CMD(cmd)](nvme, nsid, &nioc, mode, in nvme_ioctl()