Lines Matching +full:lock +full:- +full:status
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2023-2024 Chelsio Communications, Inc.
11 #include <sys/lock.h>
37 sbuf_printf(&sb, "nvmft%u: ", ctrlr->cntlid); in nvmft_printf()
56 ctrlr->cntlid = cntlid; in nvmft_controller_alloc()
57 ctrlr->np = np; in nvmft_controller_alloc()
58 mtx_init(&ctrlr->lock, "nvmft controller", NULL, MTX_DEF); in nvmft_controller_alloc()
59 callout_init(&ctrlr->ka_timer, 1); in nvmft_controller_alloc()
60 TASK_INIT(&ctrlr->shutdown_task, 0, nvmft_controller_shutdown, ctrlr); in nvmft_controller_alloc()
61 TIMEOUT_TASK_INIT(taskqueue_thread, &ctrlr->terminate_task, 0, in nvmft_controller_alloc()
64 ctrlr->cdata = np->cdata; in nvmft_controller_alloc()
65 ctrlr->cdata.ctrlr_id = htole16(cntlid); in nvmft_controller_alloc()
66 memcpy(ctrlr->hostid, data->hostid, sizeof(ctrlr->hostid)); in nvmft_controller_alloc()
67 memcpy(ctrlr->hostnqn, data->hostnqn, sizeof(ctrlr->hostnqn)); in nvmft_controller_alloc()
68 ctrlr->hip.power_cycles[0] = 1; in nvmft_controller_alloc()
69 ctrlr->create_time = sbinuptime(); in nvmft_controller_alloc()
71 ctrlr->changed_ns = malloc(sizeof(*ctrlr->changed_ns), M_NVMFT, in nvmft_controller_alloc()
80 mtx_destroy(&ctrlr->lock); in nvmft_controller_free()
81 MPASS(ctrlr->io_qpairs == NULL); in nvmft_controller_free()
82 free(ctrlr->changed_ns, M_NVMFT); in nvmft_controller_free()
92 if (ctrlr->shutdown) in nvmft_keep_alive_timer()
95 traffic = atomic_readandclear_int(&ctrlr->ka_active_traffic); in nvmft_keep_alive_timer()
103 callout_schedule_sbt(&ctrlr->ka_timer, ctrlr->ka_sbt, 0, C_HARDCLOCK); in nvmft_keep_alive_timer()
111 val = nvmft_max_ioccsz(ctrlr->admin); in nvmft_update_cdata()
113 ioccsz = le32toh(ctrlr->cdata.ioccsz) * 16; in nvmft_update_cdata()
115 ctrlr->cdata.ioccsz = htole32(val / 16); in nvmft_update_cdata()
129 if (cmd->qid != htole16(0)) in nvmft_handoff_admin_queue()
135 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_admin_queue()
139 mtx_lock(&np->lock); in nvmft_handoff_admin_queue()
140 cntlid = alloc_unr(np->ids); in nvmft_handoff_admin_queue()
141 if (cntlid == -1) { in nvmft_handoff_admin_queue()
142 mtx_unlock(&np->lock); in nvmft_handoff_admin_queue()
144 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_admin_queue()
152 TAILQ_FOREACH(ctrlr, &np->controllers, link) { in nvmft_handoff_admin_queue()
153 KASSERT(ctrlr->cntlid != cntlid, in nvmft_handoff_admin_queue()
157 mtx_unlock(&np->lock); in nvmft_handoff_admin_queue()
161 mtx_lock(&np->lock); in nvmft_handoff_admin_queue()
162 if (!np->online) { in nvmft_handoff_admin_queue()
163 mtx_unlock(&np->lock); in nvmft_handoff_admin_queue()
165 free_unr(np->ids, cntlid); in nvmft_handoff_admin_queue()
170 TAILQ_INSERT_TAIL(&np->controllers, ctrlr, link); in nvmft_handoff_admin_queue()
173 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_admin_queue()
174 ctrlr->admin = qp; in nvmft_handoff_admin_queue()
175 ctrlr->trtype = trtype; in nvmft_handoff_admin_queue()
179 * The spec requires a non-zero KeepAlive timer, but allow a in nvmft_handoff_admin_queue()
182 kato = le32toh(cmd->kato); in nvmft_handoff_admin_queue()
188 ctrlr->ka_sbt = mstosbt(roundup(kato, 1000)); in nvmft_handoff_admin_queue()
189 callout_reset_sbt(&ctrlr->ka_timer, ctrlr->ka_sbt, 0, in nvmft_handoff_admin_queue()
192 mtx_unlock(&np->lock); in nvmft_handoff_admin_queue()
209 qid = le16toh(cmd->qid); in nvmft_handoff_io_queue()
212 cntlid = le16toh(data->cntlid); in nvmft_handoff_io_queue()
218 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_io_queue()
222 mtx_lock(&np->lock); in nvmft_handoff_io_queue()
223 TAILQ_FOREACH(ctrlr, &np->controllers, link) { in nvmft_handoff_io_queue()
224 if (ctrlr->cntlid == cntlid) in nvmft_handoff_io_queue()
228 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
230 ctrlr->cntlid, qid, (int)sizeof(data->hostnqn), in nvmft_handoff_io_queue()
231 data->hostnqn); in nvmft_handoff_io_queue()
238 if (memcmp(ctrlr->hostid, data->hostid, sizeof(ctrlr->hostid)) != 0) { in nvmft_handoff_io_queue()
239 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
242 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_io_queue()
248 if (memcmp(ctrlr->hostnqn, data->hostnqn, sizeof(ctrlr->hostnqn)) != 0) { in nvmft_handoff_io_queue()
249 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
252 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_io_queue()
259 /* XXX: Require trtype == ctrlr->trtype? */ in nvmft_handoff_io_queue()
261 mtx_lock(&ctrlr->lock); in nvmft_handoff_io_queue()
262 if (ctrlr->shutdown) { in nvmft_handoff_io_queue()
263 mtx_unlock(&ctrlr->lock); in nvmft_handoff_io_queue()
264 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
267 qid, (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_io_queue()
273 if (ctrlr->num_io_queues == 0) { in nvmft_handoff_io_queue()
274 mtx_unlock(&ctrlr->lock); in nvmft_handoff_io_queue()
275 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
278 qid, (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_io_queue()
284 if (cmd->qid > ctrlr->num_io_queues) { in nvmft_handoff_io_queue()
285 mtx_unlock(&ctrlr->lock); in nvmft_handoff_io_queue()
286 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
289 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_io_queue()
295 if (ctrlr->io_qpairs[qid - 1].qp != NULL) { in nvmft_handoff_io_queue()
296 mtx_unlock(&ctrlr->lock); in nvmft_handoff_io_queue()
297 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
299 "attempt to re-create I/O queue %u from %.*s\n", qid, in nvmft_handoff_io_queue()
300 (int)sizeof(data->hostnqn), data->hostnqn); in nvmft_handoff_io_queue()
307 ctrlr->io_qpairs[qid - 1].qp = qp; in nvmft_handoff_io_queue()
308 mtx_unlock(&ctrlr->lock); in nvmft_handoff_io_queue()
309 mtx_unlock(&np->lock); in nvmft_handoff_io_queue()
326 mtx_lock(&ctrlr->lock); in nvmft_controller_shutdown()
327 for (u_int i = 0; i < ctrlr->num_io_queues; i++) { in nvmft_controller_shutdown()
328 if (ctrlr->io_qpairs[i].qp != NULL) { in nvmft_controller_shutdown()
329 ctrlr->io_qpairs[i].shutdown = true; in nvmft_controller_shutdown()
330 mtx_unlock(&ctrlr->lock); in nvmft_controller_shutdown()
331 nvmft_qpair_shutdown(ctrlr->io_qpairs[i].qp); in nvmft_controller_shutdown()
332 mtx_lock(&ctrlr->lock); in nvmft_controller_shutdown()
335 mtx_unlock(&ctrlr->lock); in nvmft_controller_shutdown()
341 mtx_lock(&ctrlr->lock); in nvmft_controller_shutdown()
342 while (ctrlr->pending_commands != 0) in nvmft_controller_shutdown()
343 mtx_sleep(&ctrlr->pending_commands, &ctrlr->lock, 0, "nvmftsh", in nvmft_controller_shutdown()
345 mtx_unlock(&ctrlr->lock); in nvmft_controller_shutdown()
348 for (u_int i = 0; i < ctrlr->num_io_queues; i++) { in nvmft_controller_shutdown()
349 if (ctrlr->io_qpairs[i].qp != NULL) in nvmft_controller_shutdown()
350 nvmft_qpair_destroy(ctrlr->io_qpairs[i].qp); in nvmft_controller_shutdown()
352 free(ctrlr->io_qpairs, M_NVMFT); in nvmft_controller_shutdown()
353 ctrlr->io_qpairs = NULL; in nvmft_controller_shutdown()
355 mtx_lock(&ctrlr->lock); in nvmft_controller_shutdown()
356 ctrlr->num_io_queues = 0; in nvmft_controller_shutdown()
359 if (NVMEV(NVME_CSTS_REG_SHST, ctrlr->csts) == NVME_SHST_OCCURRING) { in nvmft_controller_shutdown()
360 ctrlr->csts &= ~NVMEM(NVME_CSTS_REG_SHST); in nvmft_controller_shutdown()
361 ctrlr->csts |= NVMEF(NVME_CSTS_REG_SHST, NVME_SHST_COMPLETE); in nvmft_controller_shutdown()
364 if (NVMEV(NVME_CSTS_REG_CFS, ctrlr->csts) == 0) { in nvmft_controller_shutdown()
365 ctrlr->csts &= ~NVMEM(NVME_CSTS_REG_RDY); in nvmft_controller_shutdown()
366 ctrlr->shutdown = false; in nvmft_controller_shutdown()
368 mtx_unlock(&ctrlr->lock); in nvmft_controller_shutdown()
374 * (NVMe-over-Fabrics 1.1 4.6). in nvmft_controller_shutdown()
376 if (ctrlr->admin_closed || NVMEV(NVME_CSTS_REG_CFS, ctrlr->csts) != 0) in nvmft_controller_shutdown()
380 &ctrlr->terminate_task, hz * 60 * 2); in nvmft_controller_shutdown()
390 /* If the controller has been re-enabled, nothing to do. */ in nvmft_controller_terminate()
391 mtx_lock(&ctrlr->lock); in nvmft_controller_terminate()
392 if (NVMEV(NVME_CC_REG_EN, ctrlr->cc) != 0) { in nvmft_controller_terminate()
393 mtx_unlock(&ctrlr->lock); in nvmft_controller_terminate()
395 if (ctrlr->ka_sbt != 0) in nvmft_controller_terminate()
396 callout_schedule_sbt(&ctrlr->ka_timer, ctrlr->ka_sbt, 0, in nvmft_controller_terminate()
402 ctrlr->shutdown = true; in nvmft_controller_terminate()
403 mtx_unlock(&ctrlr->lock); in nvmft_controller_terminate()
405 nvmft_qpair_destroy(ctrlr->admin); in nvmft_controller_terminate()
408 np = ctrlr->np; in nvmft_controller_terminate()
409 mtx_lock(&np->lock); in nvmft_controller_terminate()
410 TAILQ_REMOVE(&np->controllers, ctrlr, link); in nvmft_controller_terminate()
411 wakeup_np = (!np->online && TAILQ_EMPTY(&np->controllers)); in nvmft_controller_terminate()
412 mtx_unlock(&np->lock); in nvmft_controller_terminate()
413 free_unr(np->ids, ctrlr->cntlid); in nvmft_controller_terminate()
417 callout_drain(&ctrlr->ka_timer); in nvmft_controller_terminate()
439 if (qp != ctrlr->admin) in nvmft_controller_error()
442 mtx_lock(&ctrlr->lock); in nvmft_controller_error()
443 if (ctrlr->shutdown) { in nvmft_controller_error()
444 ctrlr->admin_closed = true; in nvmft_controller_error()
445 mtx_unlock(&ctrlr->lock); in nvmft_controller_error()
449 if (NVMEV(NVME_CC_REG_EN, ctrlr->cc) == 0) { in nvmft_controller_error()
450 MPASS(ctrlr->num_io_queues == 0); in nvmft_controller_error()
451 mtx_unlock(&ctrlr->lock); in nvmft_controller_error()
454 * Ok to drop lock here since ctrlr->cc can't in nvmft_controller_error()
469 &ctrlr->terminate_task, NULL) == 0) in nvmft_controller_error()
471 &ctrlr->terminate_task, 0); in nvmft_controller_error()
480 ctrlr->admin_closed = true; in nvmft_controller_error()
482 mtx_lock(&ctrlr->lock); in nvmft_controller_error()
485 if (ctrlr->shutdown) { in nvmft_controller_error()
486 mtx_unlock(&ctrlr->lock); in nvmft_controller_error()
490 ctrlr->csts |= NVMEF(NVME_CSTS_REG_CFS, 1); in nvmft_controller_error()
491 ctrlr->cc &= ~NVMEM(NVME_CC_REG_EN); in nvmft_controller_error()
492 ctrlr->shutdown = true; in nvmft_controller_error()
493 mtx_unlock(&ctrlr->lock); in nvmft_controller_error()
495 callout_stop(&ctrlr->ka_timer); in nvmft_controller_error()
496 taskqueue_enqueue(taskqueue_thread, &ctrlr->shutdown_task); in nvmft_controller_error()
508 for (n = m; len > 0; n = n->m_next) { in m_getml()
509 n->m_len = M_SIZE(n); in m_getml()
510 if (n->m_len >= len) { in m_getml()
511 n->m_len = len; in m_getml()
512 MPASS(n->m_next == NULL); in m_getml()
514 len -= n->m_len; in m_getml()
527 while (m->m_len <= offset) { in m_zero()
528 offset -= m->m_len; in m_zero()
529 m = m->m_next; in m_zero()
532 todo = m->m_len - offset; in m_zero()
536 m = m->m_next; in m_zero()
537 len -= todo; in m_zero()
540 todo = m->m_len; in m_zero()
544 m = m->m_next; in m_zero()
545 len -= todo; in m_zero()
557 u_int status; in handle_get_log_page() local
561 lid = le32toh(cmd->cdw10) & 0xff; in handle_get_log_page()
562 rae = (le32toh(cmd->cdw10) & (1U << 15)) != 0; in handle_get_log_page()
563 numd = le32toh(cmd->cdw10) >> 16 | le32toh(cmd->cdw11) << 16; in handle_get_log_page()
564 offset = le32toh(cmd->cdw12) | (uint64_t)le32toh(cmd->cdw13) << 32; in handle_get_log_page()
567 status = NVME_SC_INVALID_FIELD; in handle_get_log_page()
579 m_zero(m, todo, len - todo); in handle_get_log_page()
580 status = nvmf_send_controller_data(nc, 0, m, len); in handle_get_log_page()
581 MPASS(status != NVMF_MORE); in handle_get_log_page()
588 status = NVME_SC_INVALID_FIELD; in handle_get_log_page()
591 todo = sizeof(hip) - offset; in handle_get_log_page()
595 mtx_lock(&ctrlr->lock); in handle_get_log_page()
596 hip = ctrlr->hip; in handle_get_log_page()
598 sbintime_getsec(ctrlr->busy_total) / 60; in handle_get_log_page()
600 sbintime_getsec(sbinuptime() - ctrlr->create_time) / 3600; in handle_get_log_page()
601 mtx_unlock(&ctrlr->lock); in handle_get_log_page()
606 m_zero(m, todo, len - todo); in handle_get_log_page()
607 status = nvmf_send_controller_data(nc, 0, m, len); in handle_get_log_page()
608 MPASS(status != NVMF_MORE); in handle_get_log_page()
612 if (offset >= sizeof(ctrlr->np->fp)) { in handle_get_log_page()
613 status = NVME_SC_INVALID_FIELD; in handle_get_log_page()
616 todo = sizeof(ctrlr->np->fp) - offset; in handle_get_log_page()
621 m_copyback(m, 0, todo, (char *)&ctrlr->np->fp + offset); in handle_get_log_page()
623 m_zero(m, todo, len - todo); in handle_get_log_page()
624 status = nvmf_send_controller_data(nc, 0, m, len); in handle_get_log_page()
625 MPASS(status != NVMF_MORE); in handle_get_log_page()
628 if (offset >= sizeof(*ctrlr->changed_ns)) { in handle_get_log_page()
629 status = NVME_SC_INVALID_FIELD; in handle_get_log_page()
632 todo = sizeof(*ctrlr->changed_ns) - offset; in handle_get_log_page()
637 mtx_lock(&ctrlr->lock); in handle_get_log_page()
638 m_copyback(m, 0, todo, (char *)ctrlr->changed_ns + offset); in handle_get_log_page()
639 if (offset == 0 && len == sizeof(*ctrlr->changed_ns)) in handle_get_log_page()
640 memset(ctrlr->changed_ns, 0, in handle_get_log_page()
641 sizeof(*ctrlr->changed_ns)); in handle_get_log_page()
643 ctrlr->changed_ns_reported = false; in handle_get_log_page()
644 mtx_unlock(&ctrlr->lock); in handle_get_log_page()
646 m_zero(m, todo, len - todo); in handle_get_log_page()
647 status = nvmf_send_controller_data(nc, 0, m, len); in handle_get_log_page()
648 MPASS(status != NVMF_MORE); in handle_get_log_page()
653 status = NVME_SC_INVALID_FIELD; in handle_get_log_page()
658 if (status == NVMF_SUCCESS_SENT) in handle_get_log_page()
659 nvmft_command_completed(ctrlr->admin, nc); in handle_get_log_page()
661 nvmft_send_generic_error(ctrlr->admin, nc, status); in handle_get_log_page()
668 free(m->m_ext.ext_arg1, M_NVMFT); in m_free_nslist()
677 u_int status; in handle_identify_command() local
680 cns = le32toh(cmd->cdw10) & 0xFF; in handle_identify_command()
682 if (data_len != sizeof(ctrlr->cdata)) { in handle_identify_command()
686 nvmft_send_generic_error(ctrlr->admin, nc, in handle_identify_command()
695 nvmft_dispatch_command(ctrlr->admin, nc, true); in handle_identify_command()
699 m = m_getml(sizeof(ctrlr->cdata), M_WAITOK); in handle_identify_command()
700 m_copyback(m, 0, sizeof(ctrlr->cdata), (void *)&ctrlr->cdata); in handle_identify_command()
701 status = nvmf_send_controller_data(nc, 0, m, in handle_identify_command()
702 sizeof(ctrlr->cdata)); in handle_identify_command()
703 MPASS(status != NVMF_MORE); in handle_identify_command()
711 nsid = le32toh(cmd->nsid); in handle_identify_command()
713 status = NVME_SC_INVALID_FIELD; in handle_identify_command()
718 nvmft_populate_active_nslist(ctrlr->np, nsid, nslist); in handle_identify_command()
722 m->m_len = sizeof(*nslist); in handle_identify_command()
723 status = nvmf_send_controller_data(nc, 0, m, m->m_len); in handle_identify_command()
724 MPASS(status != NVMF_MORE); in handle_identify_command()
729 status = NVME_SC_INVALID_FIELD; in handle_identify_command()
733 if (status == NVMF_SUCCESS_SENT) in handle_identify_command()
734 nvmft_command_completed(ctrlr->admin, nc); in handle_identify_command()
736 nvmft_send_generic_error(ctrlr->admin, nc, status); in handle_identify_command()
747 fid = NVMEV(NVME_FEAT_SET_FID, le32toh(cmd->cdw10)); in handle_set_features()
754 num_queues = le32toh(cmd->cdw11) & 0xffff; in handle_set_features()
761 if (le32toh(cmd->cdw11) >> 16 != num_queues) in handle_set_features()
770 mtx_lock(&ctrlr->lock); in handle_set_features()
771 if (ctrlr->num_io_queues != 0) { in handle_set_features()
772 mtx_unlock(&ctrlr->lock); in handle_set_features()
774 nvmft_send_generic_error(ctrlr->admin, nc, in handle_set_features()
780 ctrlr->num_io_queues = num_queues; in handle_set_features()
781 ctrlr->io_qpairs = io_qpairs; in handle_set_features()
782 mtx_unlock(&ctrlr->lock); in handle_set_features()
785 cqe.cdw0 = cmd->cdw11; in handle_set_features()
786 nvmft_send_response(ctrlr->admin, &cqe); in handle_set_features()
794 aer_mask = le32toh(cmd->cdw11); in handle_set_features()
800 mtx_lock(&ctrlr->lock); in handle_set_features()
801 ctrlr->aer_mask = aer_mask; in handle_set_features()
802 mtx_unlock(&ctrlr->lock); in handle_set_features()
803 nvmft_send_success(ctrlr->admin, nc); in handle_set_features()
814 nvmft_send_generic_error(ctrlr->admin, nc, NVME_SC_INVALID_FIELD); in handle_set_features()
821 struct nvmft_port *np = ctrlr->np; in update_cc()
826 mtx_lock(&ctrlr->lock); in update_cc()
829 if (ctrlr->shutdown) { in update_cc()
830 mtx_unlock(&ctrlr->lock); in update_cc()
834 if (!_nvmf_validate_cc(np->max_io_qsize, np->cap, ctrlr->cc, new_cc)) { in update_cc()
835 mtx_unlock(&ctrlr->lock); in update_cc()
839 changes = ctrlr->cc ^ new_cc; in update_cc()
840 ctrlr->cc = new_cc; in update_cc()
845 ctrlr->csts &= ~NVMEM(NVME_CSTS_REG_SHST); in update_cc()
846 ctrlr->csts |= NVMEF(NVME_CSTS_REG_SHST, NVME_SHST_OCCURRING); in update_cc()
847 ctrlr->cc &= ~NVMEM(NVME_CC_REG_EN); in update_cc()
848 ctrlr->shutdown = true; in update_cc()
857 ctrlr->shutdown = true; in update_cc()
860 ctrlr->csts |= NVMEF(NVME_CSTS_REG_RDY, 1); in update_cc()
862 mtx_unlock(&ctrlr->lock); in update_cc()
875 switch (le32toh(pget->ofst)) { in handle_property_get()
877 if (pget->attrib.size != NVMF_PROP_SIZE_8) in handle_property_get()
879 rsp.value.u64 = htole64(ctrlr->np->cap); in handle_property_get()
882 if (pget->attrib.size != NVMF_PROP_SIZE_4) in handle_property_get()
884 rsp.value.u32.low = ctrlr->cdata.ver; in handle_property_get()
887 if (pget->attrib.size != NVMF_PROP_SIZE_4) in handle_property_get()
889 rsp.value.u32.low = htole32(ctrlr->cc); in handle_property_get()
892 if (pget->attrib.size != NVMF_PROP_SIZE_4) in handle_property_get()
894 rsp.value.u32.low = htole32(ctrlr->csts); in handle_property_get()
900 nvmft_send_response(ctrlr->admin, &rsp); in handle_property_get()
903 nvmft_send_generic_error(ctrlr->admin, nc, NVME_SC_INVALID_FIELD); in handle_property_get()
913 switch (le32toh(pset->ofst)) { in handle_property_set()
915 if (pset->attrib.size != NVMF_PROP_SIZE_4) in handle_property_set()
917 if (!update_cc(ctrlr, le32toh(pset->value.u32.low), in handle_property_set()
925 nvmft_send_success(ctrlr->admin, nc); in handle_property_set()
927 callout_stop(&ctrlr->ka_timer); in handle_property_set()
928 taskqueue_enqueue(taskqueue_thread, &ctrlr->shutdown_task); in handle_property_set()
932 nvmft_send_generic_error(ctrlr->admin, nc, NVME_SC_INVALID_FIELD); in handle_property_set()
939 switch (fc->fctype) { in handle_admin_fabrics_command()
951 nvmft_send_generic_error(ctrlr->admin, nc, in handle_admin_fabrics_command()
956 nvmft_send_error(ctrlr->admin, nc, NVME_SCT_COMMAND_SPECIFIC, in handle_admin_fabrics_command()
961 fc->fctype); in handle_admin_fabrics_command()
962 nvmft_send_generic_error(ctrlr->admin, nc, in handle_admin_fabrics_command()
976 if (NVMEV(NVME_CC_REG_EN, ctrlr->cc) == 0 && in nvmft_handle_admin_command()
977 cmd->opc != NVME_OPC_FABRICS_COMMANDS) { in nvmft_handle_admin_command()
979 "Unsupported admin opcode %#x while disabled\n", cmd->opc); in nvmft_handle_admin_command()
980 nvmft_send_generic_error(ctrlr->admin, nc, in nvmft_handle_admin_command()
986 atomic_store_int(&ctrlr->ka_active_traffic, 1); in nvmft_handle_admin_command()
988 switch (cmd->opc) { in nvmft_handle_admin_command()
999 mtx_lock(&ctrlr->lock); in nvmft_handle_admin_command()
1000 if (ctrlr->aer_pending == NVMFT_NUM_AER) { in nvmft_handle_admin_command()
1001 mtx_unlock(&ctrlr->lock); in nvmft_handle_admin_command()
1002 nvmft_send_error(ctrlr->admin, nc, in nvmft_handle_admin_command()
1006 /* NB: Store the CID without byte-swapping. */ in nvmft_handle_admin_command()
1007 ctrlr->aer_cids[ctrlr->aer_pidx] = cmd->cid; in nvmft_handle_admin_command()
1008 ctrlr->aer_pending++; in nvmft_handle_admin_command()
1009 ctrlr->aer_pidx = (ctrlr->aer_pidx + 1) % NVMFT_NUM_AER; in nvmft_handle_admin_command()
1010 mtx_unlock(&ctrlr->lock); in nvmft_handle_admin_command()
1015 nvmft_send_success(ctrlr->admin, nc); in nvmft_handle_admin_command()
1023 nvmft_printf(ctrlr, "Unsupported admin opcode %#x\n", cmd->opc); in nvmft_handle_admin_command()
1024 nvmft_send_generic_error(ctrlr->admin, nc, in nvmft_handle_admin_command()
1038 atomic_store_int(&ctrlr->ka_active_traffic, 1); in nvmft_handle_io_command()
1040 switch (cmd->opc) { in nvmft_handle_io_command()
1042 if (cmd->nsid == htole32(0xffffffff)) { in nvmft_handle_io_command()
1059 nvmft_printf(ctrlr, "Unsupported I/O opcode %#x\n", cmd->opc); in nvmft_handle_io_command()
1076 mtx_lock(&ctrlr->lock); in nvmft_report_aer()
1077 if ((ctrlr->aer_mask & aer_mask) == 0) { in nvmft_report_aer()
1078 mtx_unlock(&ctrlr->lock); in nvmft_report_aer()
1086 if (ctrlr->aer_pending == 0) { in nvmft_report_aer()
1087 mtx_unlock(&ctrlr->lock); in nvmft_report_aer()
1095 cpl.cid = ctrlr->aer_cids[ctrlr->aer_cidx]; in nvmft_report_aer()
1096 ctrlr->aer_pending--; in nvmft_report_aer()
1097 ctrlr->aer_cidx = (ctrlr->aer_cidx + 1) % NVMFT_NUM_AER; in nvmft_report_aer()
1098 mtx_unlock(&ctrlr->lock); in nvmft_report_aer()
1104 nvmft_send_response(ctrlr->admin, &cpl); in nvmft_report_aer()
1116 mtx_lock(&ctrlr->lock); in nvmft_controller_lun_changed()
1117 nslist = ctrlr->changed_ns; in nvmft_controller_lun_changed()
1120 if (nslist->ns[0] != 0xffffffff) { in nvmft_controller_lun_changed()
1122 for (i = 0; i < nitems(nslist->ns); i++) { in nvmft_controller_lun_changed()
1123 nsid = le32toh(nslist->ns[i]); in nvmft_controller_lun_changed()
1126 mtx_unlock(&ctrlr->lock); in nvmft_controller_lun_changed()
1134 if (nslist->ns[nitems(nslist->ns) - 1] != htole32(0)) { in nvmft_controller_lun_changed()
1136 memset(ctrlr->changed_ns, 0, in nvmft_controller_lun_changed()
1137 sizeof(*ctrlr->changed_ns)); in nvmft_controller_lun_changed()
1138 ctrlr->changed_ns->ns[0] = 0xffffffff; in nvmft_controller_lun_changed()
1139 } else if (nslist->ns[i] == htole32(0)) { in nvmft_controller_lun_changed()
1144 nslist->ns[i] = htole32(new_nsid); in nvmft_controller_lun_changed()
1146 memmove(&nslist->ns[i + 1], &nslist->ns[i], in nvmft_controller_lun_changed()
1147 (nitems(nslist->ns) - i - 1) * in nvmft_controller_lun_changed()
1148 sizeof(nslist->ns[0])); in nvmft_controller_lun_changed()
1149 nslist->ns[i] = htole32(new_nsid); in nvmft_controller_lun_changed()
1153 if (ctrlr->changed_ns_reported) { in nvmft_controller_lun_changed()
1154 mtx_unlock(&ctrlr->lock); in nvmft_controller_lun_changed()
1157 ctrlr->changed_ns_reported = true; in nvmft_controller_lun_changed()
1158 mtx_unlock(&ctrlr->lock); in nvmft_controller_lun_changed()