Lines Matching refs:cmd
855 nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag);
857 if (cmd == NULL)
858 return (cmd);
860 bzero(cmd, sizeof (nvme_cmd_t));
862 cmd->nc_nvme = nvme;
864 mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER,
866 cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL);
868 return (cmd);
872 nvme_free_cmd(nvme_cmd_t *cmd)
875 if (list_link_active(&cmd->nc_list))
878 if (cmd->nc_dma) {
879 if (cmd->nc_dma->nd_cached)
880 kmem_cache_free(cmd->nc_nvme->n_prp_cache,
881 cmd->nc_dma);
883 nvme_free_dma(cmd->nc_dma);
884 cmd->nc_dma = NULL;
887 cv_destroy(&cmd->nc_cv);
888 mutex_destroy(&cmd->nc_mutex);
890 kmem_cache_free(nvme_cmd_cache, cmd);
894 nvme_submit_admin_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd)
897 nvme_submit_cmd_common(qp, cmd);
901 nvme_submit_io_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd)
906 nvme_submit_cmd_common(qp, cmd);
911 nvme_submit_cmd_common(nvme_qpair_t *qp, nvme_cmd_t *cmd)
916 cmd->nc_completed = B_FALSE;
919 * Try to insert the cmd into the active cmd array at the nq_next_cmd
926 qp->nq_cmd[qp->nq_next_cmd] = cmd;
930 cmd->nc_sqe.sqe_cid = qp->nq_next_cmd;
931 bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t));
938 nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r);
946 nvme_cmd_t *cmd;
951 cmd = qp->nq_cmd[cid];
957 ASSERT3P(cmd, !=, NULL);
958 ASSERT3P(cmd->nc_nvme, ==, nvme);
959 ASSERT3S(cmd->nc_sqe.sqe_cid, ==, cid);
961 return (cmd);
970 nvme_cmd_t *cmd;
986 cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid);
988 ASSERT(cmd->nc_sqid == cqe->cqe_sqid);
989 bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t));
999 nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r);
1002 return (cmd);
1006 nvme_check_unknown_cmd_status(nvme_cmd_t *cmd)
1008 nvme_cqe_t *cqe = &cmd->nc_cqe;
1010 dev_err(cmd->nc_nvme->n_dip, CE_WARN,
1012 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc,
1016 if (cmd->nc_xfer != NULL)
1017 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
1019 if (cmd->nc_nvme->n_strict_version) {
1020 cmd->nc_nvme->n_dead = B_TRUE;
1021 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
1028 nvme_check_vendor_cmd_status(nvme_cmd_t *cmd)
1030 nvme_cqe_t *cqe = &cmd->nc_cqe;
1032 dev_err(cmd->nc_nvme->n_dip, CE_WARN,
1034 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc,
1037 if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) {
1038 cmd->nc_nvme->n_dead = B_TRUE;
1039 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
1046 nvme_check_integrity_cmd_status(nvme_cmd_t *cmd)
1048 nvme_cqe_t *cqe = &cmd->nc_cqe;
1054 if (cmd->nc_xfer != NULL)
1055 bd_error(cmd->nc_xfer, BD_ERR_MEDIA);
1061 if (cmd->nc_xfer != NULL)
1062 bd_error(cmd->nc_xfer, BD_ERR_MEDIA);
1066 return (nvme_check_unknown_cmd_status(cmd));
1071 nvme_check_generic_cmd_status(nvme_cmd_t *cmd)
1073 nvme_cqe_t *cqe = &cmd->nc_cqe;
1084 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
1085 "invalid opcode in cmd %p", (void *)cmd);
1090 if (!cmd->nc_dontpanic)
1091 dev_err(cmd->nc_nvme->n_dip, CE_PANIC,
1092 "programming error: invalid field in cmd %p",
1093 (void *)cmd);
1098 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
1099 "cmd ID conflict in cmd %p", (void *)cmd);
1104 if (!cmd->nc_dontpanic)
1105 dev_err(cmd->nc_nvme->n_dip, CE_PANIC,
1106 "programming error: " "invalid NS/format in cmd %p",
1107 (void *)cmd);
1112 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
1113 "LBA out of range in cmd %p", (void *)cmd);
1122 atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err);
1123 if (cmd->nc_xfer != NULL)
1124 bd_error(cmd->nc_xfer, BD_ERR_NTRDY);
1134 atomic_inc_32(&cmd->nc_nvme->n_internal_err);
1135 if (cmd->nc_xfer != NULL)
1136 bd_error(cmd->nc_xfer, BD_ERR_NTRDY);
1145 atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err);
1150 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST);
1151 cmd->nc_nvme->n_dead = B_TRUE;
1156 atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del);
1161 atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc);
1162 if (cmd->nc_xfer != NULL)
1163 bd_error(cmd->nc_xfer, BD_ERR_MEDIA);
1168 atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy);
1169 if (cmd->nc_xfer != NULL)
1170 bd_error(cmd->nc_xfer, BD_ERR_NTRDY);
1174 return (nvme_check_unknown_cmd_status(cmd));
1179 nvme_check_specific_cmd_status(nvme_cmd_t *cmd)
1181 nvme_cqe_t *cqe = &cmd->nc_cqe;
1186 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE);
1187 atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err);
1192 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE ||
1193 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE ||
1194 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE ||
1195 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE);
1196 atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err);
1201 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE ||
1202 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE);
1203 atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc);
1208 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT);
1209 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
1210 "abort command limit exceeded in cmd %p", (void *)cmd);
1215 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT);
1216 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: "
1217 "async event request limit exceeded in cmd %p",
1218 (void *)cmd);
1223 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE);
1224 atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect);
1229 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE);
1230 atomic_inc_32(&cmd->nc_nvme->n_inv_log_page);
1235 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT);
1236 atomic_inc_32(&cmd->nc_nvme->n_inv_format);
1237 if (cmd->nc_xfer != NULL)
1238 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
1243 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE);
1244 atomic_inc_32(&cmd->nc_nvme->n_inv_q_del);
1249 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT ||
1250 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ ||
1251 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE);
1252 atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr);
1253 if (cmd->nc_xfer != NULL)
1254 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
1259 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE ||
1260 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ ||
1261 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE);
1262 atomic_inc_32(&cmd->nc_nvme->n_inv_prot);
1263 if (cmd->nc_xfer != NULL)
1264 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
1269 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE);
1270 atomic_inc_32(&cmd->nc_nvme->n_readonly);
1271 if (cmd->nc_xfer != NULL)
1272 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ);
1276 return (nvme_check_unknown_cmd_status(cmd));
1281 nvme_check_cmd_status(nvme_cmd_t *cmd)
1283 nvme_cqe_t *cqe = &cmd->nc_cqe;
1289 if (cmd->nc_nvme->n_dead)
1297 return (nvme_check_generic_cmd_status(cmd));
1299 return (nvme_check_specific_cmd_status(cmd));
1301 return (nvme_check_integrity_cmd_status(cmd));
1303 return (nvme_check_vendor_cmd_status(cmd));
1305 return (nvme_check_unknown_cmd_status(cmd));
1312 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1321 cmd->nc_sqid = 0;
1322 cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT;
1323 cmd->nc_callback = nvme_wakeup_cmd;
1324 cmd->nc_sqe.sqe_cdw10 = ac.r;
1332 nvme_admin_cmd(cmd, sec);
1335 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
1338 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1344 cmd->nc_cqe.cqe_dw0 & 1 ? "un" : "");
1345 if ((cmd->nc_cqe.cqe_dw0 & 1) == 0)
1349 nvme_free_cmd(cmd);
1360 nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec)
1363 nvme_t *nvme = cmd->nc_nvme;
1367 ASSERT(mutex_owned(&cmd->nc_mutex));
1369 while (!cmd->nc_completed) {
1370 if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1)
1374 if (cmd->nc_completed)
1386 "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_cid, cmd->nc_sqid,
1387 cmd->nc_sqe.sqe_opc, csts.b.csts_cfs);
1392 nvme_check_dma_hdl(cmd->nc_dma) ||
1393 cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) {
1396 } else if (nvme_abort_cmd(cmd, sec) == 0) {
1401 while (!cmd->nc_completed)
1402 cv_wait(&cmd->nc_cv, &cmd->nc_mutex);
1407 qp = nvme->n_ioq[cmd->nc_sqid];
1410 (void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid);
1418 if (cmd->nc_dma != NULL) {
1420 list_insert_head(&nvme_lost_cmds, cmd);
1428 nvme_cmd_t *cmd = arg;
1430 mutex_enter(&cmd->nc_mutex);
1431 cmd->nc_completed = B_TRUE;
1432 cv_signal(&cmd->nc_cv);
1433 mutex_exit(&cmd->nc_mutex);
1439 nvme_cmd_t *cmd = arg;
1440 nvme_t *nvme = cmd->nc_nvme;
1456 if (nvme_check_cmd_status(cmd) != 0) {
1457 dev_err(cmd->nc_nvme->n_dip, CE_WARN,
1459 "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct,
1460 cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr,
1461 cmd->nc_cqe.cqe_sf.sf_m);
1463 if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC &&
1464 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) {
1465 cmd->nc_nvme->n_dead = B_TRUE;
1466 ddi_fm_service_impact(cmd->nc_nvme->n_dip,
1469 nvme_free_cmd(cmd);
1474 event.r = cmd->nc_cqe.cqe_dw0;
1477 bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t));
1478 nvme_submit_admin_cmd(nvme->n_adminq, cmd);
1589 nvme_admin_cmd(nvme_cmd_t *cmd, int sec)
1591 mutex_enter(&cmd->nc_mutex);
1592 nvme_submit_admin_cmd(cmd->nc_nvme->n_adminq, cmd);
1593 nvme_wait_cmd(cmd, sec);
1594 mutex_exit(&cmd->nc_mutex);
1600 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1602 cmd->nc_sqid = 0;
1603 cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT;
1604 cmd->nc_callback = nvme_async_event_task;
1606 nvme_submit_admin_cmd(nvme->n_adminq, cmd);
1613 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1623 cmd->nc_sqid = 0;
1624 cmd->nc_callback = nvme_wakeup_cmd;
1625 cmd->nc_sqe.sqe_nsid = nsid;
1626 cmd->nc_sqe.sqe_opc = NVME_OPC_NVM_FORMAT;
1627 cmd->nc_sqe.sqe_cdw10 = format_nvm.r;
1634 cmd->nc_dontpanic = B_TRUE;
1636 nvme_admin_cmd(cmd, nvme_format_cmd_timeout);
1638 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
1641 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1644 nvme_free_cmd(cmd);
1652 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1659 cmd->nc_sqid = 0;
1660 cmd->nc_callback = nvme_wakeup_cmd;
1661 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE;
1667 cmd->nc_sqe.sqe_nsid = (uint32_t)-1;
1677 cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t);
1682 cmd->nc_sqe.sqe_nsid = (uint32_t)-1;
1698 cmd->nc_sqe.sqe_cdw10 = getlogpage.r;
1701 DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
1708 if (cmd->nc_dma->nd_ncookie > 2) {
1716 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress;
1717 if (cmd->nc_dma->nd_ncookie > 1) {
1718 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah,
1719 &cmd->nc_dma->nd_cookie);
1720 cmd->nc_sqe.sqe_dptr.d_prp[1] =
1721 cmd->nc_dma->nd_cookie.dmac_laddress;
1724 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout);
1726 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
1729 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1734 bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize);
1737 nvme_free_cmd(cmd);
1745 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1751 cmd->nc_sqid = 0;
1752 cmd->nc_callback = nvme_wakeup_cmd;
1753 cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY;
1754 cmd->nc_sqe.sqe_nsid = nsid;
1755 cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL;
1758 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
1765 if (cmd->nc_dma->nd_ncookie > 2) {
1773 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress;
1774 if (cmd->nc_dma->nd_ncookie > 1) {
1775 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah,
1776 &cmd->nc_dma->nd_cookie);
1777 cmd->nc_sqe.sqe_dptr.d_prp[1] =
1778 cmd->nc_dma->nd_cookie.dmac_laddress;
1781 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout);
1783 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
1786 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
1791 bcopy(cmd->nc_dma->nd_memp, *buf, NVME_IDENTIFY_BUFSIZE);
1794 nvme_free_cmd(cmd);
1804 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1809 cmd->nc_sqid = 0;
1810 cmd->nc_callback = nvme_wakeup_cmd;
1811 cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES;
1812 cmd->nc_sqe.sqe_cdw10 = feature;
1813 cmd->nc_sqe.sqe_cdw11 = val;
1828 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout);
1830 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
1833 feature, cmd->nc_cqe.cqe_sf.sf_sct,
1834 cmd->nc_cqe.cqe_sf.sf_sc);
1838 *res = cmd->nc_cqe.cqe_dw0;
1841 nvme_free_cmd(cmd);
1849 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
1857 cmd->nc_sqid = 0;
1858 cmd->nc_callback = nvme_wakeup_cmd;
1859 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_FEATURES;
1860 cmd->nc_sqe.sqe_cdw10 = feature;
1861 cmd->nc_sqe.sqe_cdw11 = *res;
1892 cmd->nc_dontpanic = B_TRUE;
1893 cmd->nc_sqe.sqe_nsid = nsid;
1913 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
1920 if (cmd->nc_dma->nd_ncookie > 2) {
1928 cmd->nc_sqe.sqe_dptr.d_prp[0] =
1929 cmd->nc_dma->nd_cookie.dmac_laddress;
1930 if (cmd->nc_dma->nd_ncookie > 1) {
1931 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah,
1932 &cmd->nc_dma->nd_cookie);
1933 cmd->nc_sqe.sqe_dptr.d_prp[1] =
1934 cmd->nc_dma->nd_cookie.dmac_laddress;
1938 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout);
1940 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
1942 cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC &&
1943 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_FLD)
1948 feature, cmd->nc_cqe.cqe_sf.sf_sct,
1949 cmd->nc_cqe.cqe_sf.sf_sc);
1956 bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize);
1959 *res = cmd->nc_cqe.cqe_dw0;
1962 nvme_free_cmd(cmd);
2003 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
2016 cmd->nc_sqid = 0;
2017 cmd->nc_callback = nvme_wakeup_cmd;
2018 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE;
2019 cmd->nc_sqe.sqe_cdw10 = dw10.r;
2020 cmd->nc_sqe.sqe_cdw11 = c_dw11.r;
2021 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress;
2023 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout);
2025 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
2028 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
2032 nvme_free_cmd(cmd);
2037 cmd = nvme_alloc_cmd(nvme, KM_SLEEP);
2038 cmd->nc_sqid = 0;
2039 cmd->nc_callback = nvme_wakeup_cmd;
2040 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE;
2041 cmd->nc_sqe.sqe_cdw10 = dw10.r;
2042 cmd->nc_sqe.sqe_cdw11 = s_dw11.r;
2043 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress;
2045 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout);
2047 if ((ret = nvme_check_cmd_status(cmd)) != 0) {
2050 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc);
2055 nvme_free_cmd(cmd);
2645 nvme_cmd_t *cmd;
2662 while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) {
2663 taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq,
2664 cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent);
2807 nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2816 if (cmd != DDI_ATTACH)
2930 dev_err(dip, CE_WARN, "!failed to create cmd taskq");
3000 nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
3005 if (cmd != DDI_DETACH)
3125 nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer)
3127 nvme_t *nvme = cmd->nc_nvme;
3134 cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress;
3138 cmd->nc_sqe.sqe_dptr.d_prp[1] = 0;
3141 cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress;
3158 cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP);
3159 bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len);
3161 cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress;
3164 for (prp = (uint64_t *)cmd->nc_dma->nd_memp;
3171 (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len,
3180 nvme_cmd_t *cmd;
3185 cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ?
3188 if (cmd == NULL)
3191 cmd->nc_sqe.sqe_opc = opc;
3192 cmd->nc_callback = nvme_bd_xfer_done;
3193 cmd->nc_xfer = xfer;
3200 cmd->nc_sqe.sqe_nsid = ns->ns_id;
3202 cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu;
3203 cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32);
3204 cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1);
3206 if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS)
3211 cmd->nc_sqe.sqe_nsid = ns->ns_id;
3218 return (cmd);
3221 nvme_free_cmd(cmd);
3228 nvme_cmd_t *cmd = arg;
3229 bd_xfer_t *xfer = cmd->nc_xfer;
3232 error = nvme_check_cmd_status(cmd);
3233 nvme_free_cmd(cmd);
3296 nvme_cmd_t *cmd;
3304 cmd = nvme_create_nvm_cmd(ns, opc, xfer);
3305 if (cmd == NULL)
3308 cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1;
3309 ASSERT(cmd->nc_sqid <= nvme->n_ioq_count);
3310 ioq = nvme->n_ioq[cmd->nc_sqid];
3315 * treat both cmd and xfer as if they have been freed already.
3319 ret = nvme_submit_io_cmd(ioq, cmd);
3328 cmd = nvme_retrieve_cmd(nvme, ioq);
3329 if (cmd != NULL)
3330 nvme_bd_xfer_done(cmd);
3820 nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p,
3852 if (IS_DEVCTL(cmd))
3853 return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0));
3877 if (nvme->n_dead && cmd != NVME_IOC_DETACH)
3881 if (cmd == NVME_IOC_IDENTIFY_CTRL) {
3887 } else if (cmd == NVME_IOC_IDENTIFY_NSID && nsid == 0) {
3895 if (IS_NVME_IOC(cmd) && nvme_ioctl[NVME_IOC_CMD(cmd)] != NULL)
3896 rv = nvme_ioctl[NVME_IOC_CMD(cmd)](nvme, nsid, &nioc, mode,