1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
4  * Copyright (c) 2023-2024 Chelsio Communications, Inc.
13 #include <sys/lock.h>
45 #define	NVMFT_NC(io)	((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[0])
46 #define	NVMFT_QP(io)	((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[1])
73 	mtx_lock(&np->lock);  in nvmft_online()
74 	np->online = true;  in nvmft_online()
75 	mtx_unlock(&np->lock);  in nvmft_online()
84 	mtx_lock(&np->lock);  in nvmft_offline()
85 	np->online = false;  in nvmft_offline()
87 	TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_offline()
93 	while (!TAILQ_EMPTY(&np->controllers))  in nvmft_offline()
94 		mtx_sleep(np, &np->lock, 0, "nvmfoff", 0);  in nvmft_offline()
95 	mtx_unlock(&np->lock);  in nvmft_offline()
105 	mtx_lock(&np->lock);  in nvmft_info()
106 	retval = sbuf_printf(sb, "\t<port>%s,p,%u</port>\n", np->cdata.subnqn,  in nvmft_info()
107 	    np->portid);  in nvmft_info()
111 	TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_info()
113 		    ctrlr->cntlid, ctrlr->hostnqn);  in nvmft_info()
118 	mtx_unlock(&np->lock);  in nvmft_info()
131 	if (lun_id >= le32toh(np->cdata.nn)) {  in nvmft_lun_enable()
133 		    np->cdata.subnqn, lun_id, le32toh(np->cdata.nn));  in nvmft_lun_enable()
138 	mtx_lock(&np->lock);  in nvmft_lun_enable()
140 		new_count = np->num_ns + 1;  in nvmft_lun_enable()
141 		mtx_unlock(&np->lock);  in nvmft_lun_enable()
144 		mtx_lock(&np->lock);  in nvmft_lun_enable()
145 		if (np->num_ns + 1 <= new_count)  in nvmft_lun_enable()
149 	for (i = 0; i < np->num_ns; i++) {  in nvmft_lun_enable()
150 		if (np->active_ns[i] < nsid)  in nvmft_lun_enable()
152 		if (np->active_ns[i] == nsid) {  in nvmft_lun_enable()
153 			mtx_unlock(&np->lock);  in nvmft_lun_enable()
156 			    np->cdata.subnqn, lun_id);  in nvmft_lun_enable()
163 	memcpy(new_ns, np->active_ns, i * sizeof(*np->active_ns));  in nvmft_lun_enable()
169 	memcpy(new_ns + i + 1, np->active_ns + i, (np->num_ns - i) *  in nvmft_lun_enable()
170 	    sizeof(*np->active_ns));  in nvmft_lun_enable()
172 	np->num_ns++;  in nvmft_lun_enable()
173 	old_ns = np->active_ns;  in nvmft_lun_enable()
174 	np->active_ns = new_ns;  in nvmft_lun_enable()
176 	TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_lun_enable()
180 	mtx_unlock(&np->lock);  in nvmft_lun_enable()
194 	if (lun_id >= le32toh(np->cdata.nn))  in nvmft_lun_disable()
198 	mtx_lock(&np->lock);  in nvmft_lun_disable()
199 	for (i = 0; i < np->num_ns; i++) {  in nvmft_lun_disable()
200 		if (np->active_ns[i] == nsid)  in nvmft_lun_disable()
203 	mtx_unlock(&np->lock);  in nvmft_lun_disable()
205 	    np->cdata.subnqn, lun_id);  in nvmft_lun_disable()
210 	memmove(np->active_ns + i, np->active_ns + i + 1,  in nvmft_lun_disable()
211 	    (np->num_ns - (i + 1)) * sizeof(*np->active_ns));  in nvmft_lun_disable()
212 	np->num_ns--;  in nvmft_lun_disable()
216 	TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_lun_disable()
220 	mtx_unlock(&np->lock);  in nvmft_lun_disable()
231 	mtx_lock(&np->lock);  in nvmft_populate_active_nslist()
233 	for (i = 0; i < np->num_ns; i++) {  in nvmft_populate_active_nslist()
234 		if (np->active_ns[i] <= nsid)  in nvmft_populate_active_nslist()
236 		nslist->ns[count] = htole32(np->active_ns[i]);  in nvmft_populate_active_nslist()
238 		if (count == nitems(nslist->ns))  in nvmft_populate_active_nslist()
241 	mtx_unlock(&np->lock);  in nvmft_populate_active_nslist()
250 	struct nvmft_port *np = ctrlr->np;  in nvmft_dispatch_command()
254 	if (cmd->nsid == htole32(0)) {  in nvmft_dispatch_command()
261 	mtx_lock(&ctrlr->lock);  in nvmft_dispatch_command()
262 	if (ctrlr->pending_commands == 0)  in nvmft_dispatch_command()
263 		ctrlr->start_busy = sbinuptime();  in nvmft_dispatch_command()
264 	ctrlr->pending_commands++;  in nvmft_dispatch_command()
265 	mtx_unlock(&ctrlr->lock);  in nvmft_dispatch_command()
266 	io = ctl_alloc_io(np->port.ctl_pool_ref);  in nvmft_dispatch_command()
270 	io->io_hdr.io_type = admin ? CTL_IO_NVME_ADMIN : CTL_IO_NVME;  in nvmft_dispatch_command()
271 	io->io_hdr.nexus.initid = ctrlr->cntlid;  in nvmft_dispatch_command()
272 	io->io_hdr.nexus.targ_port = np->port.targ_port;  in nvmft_dispatch_command()
273 	io->io_hdr.nexus.targ_lun = le32toh(cmd->nsid) - 1;  in nvmft_dispatch_command()
274 	io->nvmeio.cmd = *cmd;  in nvmft_dispatch_command()
279 		ctl_nvme_set_generic_error(&io->nvmeio,  in nvmft_dispatch_command()
290 	struct nvmft_port *np = ctrlr->np;  in nvmft_terminate_commands()
294 	mtx_lock(&ctrlr->lock);  in nvmft_terminate_commands()
295 	if (ctrlr->pending_commands == 0)  in nvmft_terminate_commands()
296 		ctrlr->start_busy = sbinuptime();  in nvmft_terminate_commands()
297 	ctrlr->pending_commands++;  in nvmft_terminate_commands()
298 	mtx_unlock(&ctrlr->lock);  in nvmft_terminate_commands()
299 	io = ctl_alloc_io(np->port.ctl_pool_ref);  in nvmft_terminate_commands()
301 	NVMFT_QP(io) = ctrlr->admin;  in nvmft_terminate_commands()
302 	io->io_hdr.io_type = CTL_IO_TASK;  in nvmft_terminate_commands()
303 	io->io_hdr.nexus.initid = ctrlr->cntlid;  in nvmft_terminate_commands()
304 	io->io_hdr.nexus.targ_port = np->port.targ_port;  in nvmft_terminate_commands()
305 	io->io_hdr.nexus.targ_lun = 0;  in nvmft_terminate_commands()
306 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX: unused? */  in nvmft_terminate_commands()
307 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;  in nvmft_terminate_commands()
312 		io->io_hdr.status = CTL_SUCCESS;  in nvmft_terminate_commands()
326 		MPASS(xfered == ctnio->kern_data_len);  in nvmft_datamove_out_cb()
327 		ctnio->kern_data_resid -= xfered;  in nvmft_datamove_out_cb()
330 	if (ctnio->kern_sg_entries) {  in nvmft_datamove_out_cb()
331 		free(ctnio->ext_data_ptr, M_NVMFT);  in nvmft_datamove_out_cb()
332 		ctnio->ext_data_ptr = NULL;  in nvmft_datamove_out_cb()
334 		MPASS(ctnio->ext_data_ptr == NULL);  in nvmft_datamove_out_cb()
345 	MPASS(ctnio->ext_data_ptr == NULL);  in nvmft_datamove_out()
346 	if (ctnio->kern_sg_entries > 0) {  in nvmft_datamove_out()
350 		vlist = mallocarray(ctnio->kern_sg_entries, sizeof(*vlist),  in nvmft_datamove_out()
352 		ctnio->ext_data_ptr = (void *)vlist;  in nvmft_datamove_out()
353 		sgl = (struct ctl_sg_entry *)ctnio->kern_data_ptr;  in nvmft_datamove_out()
354 		for (u_int i = 0; i < ctnio->kern_sg_entries; i++) {  in nvmft_datamove_out()
358 		mem = memdesc_vlist(vlist, ctnio->kern_sg_entries);  in nvmft_datamove_out()
360 		mem = memdesc_vaddr(ctnio->kern_data_ptr, ctnio->kern_data_len);  in nvmft_datamove_out()
362 	error = nvmf_receive_controller_data(nc, ctnio->kern_rel_offset, &mem,  in nvmft_datamove_out()
363 	    ctnio->kern_data_len, nvmft_datamove_out_cb, ctnio);  in nvmft_datamove_out()
371 	if (ctnio->kern_sg_entries) {  in nvmft_datamove_out()
372 		free(ctnio->ext_data_ptr, M_NVMFT);  in nvmft_datamove_out()
373 		ctnio->ext_data_ptr = NULL;  in nvmft_datamove_out()
375 		MPASS(ctnio->ext_data_ptr == NULL);  in nvmft_datamove_out()
387 	MPASS(ctnio->kern_data_len != 0);  in nvmft_copy_data()
389 	m0 = m_getm2(NULL, ctnio->kern_data_len, M_WAITOK, MT_DATA, 0);  in nvmft_copy_data()
391 	if (ctnio->kern_sg_entries == 0) {  in nvmft_copy_data()
392 		m_copyback(m0, 0, ctnio->kern_data_len, ctnio->kern_data_ptr);  in nvmft_copy_data()
396 	resid = ctnio->kern_data_len;  in nvmft_copy_data()
397 	sgl = (struct ctl_sg_entry *)ctnio->kern_data_ptr;  in nvmft_copy_data()
402 		todo = MIN(mlen, sgl->len - off);  in nvmft_copy_data()
403 		memcpy(mtod(m, char *) + m->m_len, (char *)sgl->addr + off,  in nvmft_copy_data()
405 		m->m_len += todo;  in nvmft_copy_data()
406 		resid -= todo;  in nvmft_copy_data()
408 			MPASS(m->m_next == NULL);  in nvmft_copy_data()
413 		if (off == sgl->len) {  in nvmft_copy_data()
417 		mlen -= todo;  in nvmft_copy_data()
419 			m = m->m_next;  in nvmft_copy_data()
430 	ctl_ref kern_data_ref = m->m_ext.ext_arg1;  in m_free_ref_data()
432 	kern_data_ref(m->m_ext.ext_arg2, -1);  in m_free_ref_data()
441 	m_extadd(m, buf, size, m_free_ref_data, ctnio->kern_data_ref,  in m_get_ref_data()
442 	    ctnio->kern_data_arg, M_RDONLY, EXT_CTL);  in m_get_ref_data()
443 	m->m_len = size;  in m_get_ref_data()
444 	ctnio->kern_data_ref(ctnio->kern_data_arg, 1);  in m_get_ref_data()
454 	MPASS(ctnio->kern_data_len != 0);  in nvmft_ref_data()
456 	if (ctnio->kern_sg_entries == 0)  in nvmft_ref_data()
457 		return (m_get_ref_data(ctnio, ctnio->kern_data_ptr,  in nvmft_ref_data()
458 		    ctnio->kern_data_len));  in nvmft_ref_data()
460 	sgl = (struct ctl_sg_entry *)ctnio->kern_data_ptr;  in nvmft_ref_data()
463 	for (u_int i = 1; i < ctnio->kern_sg_entries; i++) {  in nvmft_ref_data()
464 		m->m_next = m_get_ref_data(ctnio, sgl[i].addr, sgl[i].len);  in nvmft_ref_data()
465 		m = m->m_next;  in nvmft_ref_data()
475 	u_int status;  in nvmft_datamove_in()  local
477 	if (ctnio->kern_data_ref != NULL)  in nvmft_datamove_in()
481 	status = nvmf_send_controller_data(nc, ctnio->kern_rel_offset, m,  in nvmft_datamove_in()
482 	    ctnio->kern_data_len);  in nvmft_datamove_in()
483 	switch (status) {  in nvmft_datamove_in()
485 		ctnio->success_sent = true;  in nvmft_datamove_in()
492 		ctl_nvme_set_generic_error(ctnio, status);  in nvmft_datamove_in()
504 	/* Some CTL commands preemptively set a success status. */  in nvmft_handle_datamove()
505 	MPASS(io->io_hdr.status == CTL_STATUS_NONE ||  in nvmft_handle_datamove()
506 	    io->io_hdr.status == CTL_SUCCESS);  in nvmft_handle_datamove()
507 	MPASS(!io->nvmeio.success_sent);  in nvmft_handle_datamove()
512 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)  in nvmft_handle_datamove()
513 		nvmft_datamove_in(&io->nvmeio, qp, nc);  in nvmft_handle_datamove()
515 		nvmft_datamove_out(&io->nvmeio, qp, nc);  in nvmft_handle_datamove()
521 	io->io_hdr.port_status = 1;  in nvmft_abort_datamove()
522 	io->io_hdr.flags |= CTL_FLAG_ABORT;  in nvmft_abort_datamove()
568 	KASSERT(io->io_hdr.status == CTL_SUCCESS ||  in nvmft_done()
569 	    io->io_hdr.status == CTL_NVME_ERROR,  in nvmft_done()
570 	    ("%s: bad status %u", __func__, io->io_hdr.status));  in nvmft_done()
583 	if (io->io_hdr.status == CTL_SUCCESS)  in nvmft_done()
587 	switch (cmd->opc) {  in nvmft_done()
589 		mtx_lock(&ctrlr->lock);  in nvmft_done()
590 		hip_add(ctrlr->hip.host_write_commands, 1);  in nvmft_done()
591 		len += ctrlr->partial_duw;  in nvmft_done()
593 			hip_add(ctrlr->hip.data_units_written, len / 1000);  in nvmft_done()
594 		ctrlr->partial_duw = len % 1000;  in nvmft_done()
595 		mtx_unlock(&ctrlr->lock);  in nvmft_done()
600 		mtx_lock(&ctrlr->lock);  in nvmft_done()
601 		if (cmd->opc != NVME_OPC_VERIFY)  in nvmft_done()
602 			hip_add(ctrlr->hip.host_read_commands, 1);  in nvmft_done()
603 		len += ctrlr->partial_dur;  in nvmft_done()
605 			hip_add(ctrlr->hip.data_units_read, len / 1000);  in nvmft_done()
606 		ctrlr->partial_dur = len % 1000;  in nvmft_done()
607 		mtx_unlock(&ctrlr->lock);  in nvmft_done()
611 	if (io->nvmeio.success_sent) {  in nvmft_done()
612 		MPASS(io->io_hdr.status == CTL_SUCCESS);  in nvmft_done()
614 		io->nvmeio.cpl.cid = cmd->cid;  in nvmft_done()
615 		nvmft_send_response(qp, &io->nvmeio.cpl);  in nvmft_done()
620 	mtx_lock(&ctrlr->lock);  in nvmft_done()
621 	ctrlr->pending_commands--;  in nvmft_done()
622 	if (ctrlr->pending_commands == 0)  in nvmft_done()
623 		ctrlr->busy_total += sbinuptime() - ctrlr->start_busy;  in nvmft_done()
624 	mtx_unlock(&ctrlr->lock);  in nvmft_done()
635 	    control_softc->ctl_proc, "nvmft");  in nvmft_init()
649 	KASSERT(TAILQ_EMPTY(&np->controllers),  in nvmft_port_free()
652 	if (np->port.targ_port != -1) {  in nvmft_port_free()
653 		if (ctl_port_deregister(&np->port) != 0)  in nvmft_port_free()
657 	free(np->active_ns, M_NVMFT);  in nvmft_port_free()
658 	clean_unrhdr(np->ids);  in nvmft_port_free()
659 	delete_unrhdr(np->ids);  in nvmft_port_free()
660 	mtx_destroy(&np->lock);  in nvmft_port_free()
673 		if (strcmp(np->cdata.subnqn, subnqn) == 0)  in nvmft_port_find()
686 		if (np->port.targ_port == port_id)  in nvmft_port_find_by_id()
743 	subnqn = dnvlist_get_string(req->args_nvl, "subnqn", NULL);  in nvmft_port_create()
744 	if (subnqn == NULL || !nvlist_exists_string(req->args_nvl, "portid")) {  in nvmft_port_create()
745 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
746 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
751 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
752 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
756 	if (!dnvlist_get_strnum(req->args_nvl, "portid", UINT16_MAX, &portid) ||  in nvmft_port_create()
758 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
759 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
765 	if (!dnvlist_get_strnum(req->args_nvl, "max_io_qsize",  in nvmft_port_create()
769 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
770 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
775 	if (!dnvlist_get_strnum(req->args_nvl, "enable_timeout",  in nvmft_port_create()
778 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
779 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
784 	if (!dnvlist_get_strnum(req->args_nvl, "ioccsz", NVMF_IOCCSZ,  in nvmft_port_create()
787 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
788 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
793 	if (!dnvlist_get_strnum(req->args_nvl, "iorcsz", NVMF_IORCSZ,  in nvmft_port_create()
796 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
797 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
802 	if (!dnvlist_get_strnum(req->args_nvl, "nn", NVMF_NN, &nn) ||  in nvmft_port_create()
804 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
805 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
810 	serial = dnvlist_get_string(req->args_nvl, "serial", NULL);  in nvmft_port_create()
812 		getcredhostid(curthread->td_ucred, &hostid);  in nvmft_port_create()
821 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
822 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
829 	refcount_init(&np->refs, 1);  in nvmft_port_create()
830 	np->portid = portid;  in nvmft_port_create()
831 	np->max_io_qsize = max_io_qsize;  in nvmft_port_create()
832 	np->cap = _nvmf_controller_cap(max_io_qsize, enable_timeout / 500);  in nvmft_port_create()
833 	mtx_init(&np->lock, "nvmft port", NULL, MTX_DEF);  in nvmft_port_create()
834 	np->ids = new_unrhdr(0, MIN(CTL_MAX_INIT_PER_PORT - 1,  in nvmft_port_create()
836 	TAILQ_INIT(&np->controllers);  in nvmft_port_create()
840 	    osrelease, subnqn, nn, ioccsz, iorcsz, &np->cdata);  in nvmft_port_create()
841 	np->cdata.aerl = NVMFT_NUM_AER - 1;  in nvmft_port_create()
842 	np->cdata.oaes = htole32(NVME_ASYNC_EVENT_NS_ATTRIBUTE);  in nvmft_port_create()
843 	np->cdata.oncs = htole16(NVMEF(NVME_CTRLR_DATA_ONCS_VERIFY, 1) |  in nvmft_port_create()
847 	np->cdata.fuses = NVMEF(NVME_CTRLR_DATA_FUSES_CNW, 1);  in nvmft_port_create()
849 	np->fp.afi = NVMEF(NVME_FIRMWARE_PAGE_AFI_SLOT, 1);  in nvmft_port_create()
850 	memcpy(np->fp.revision[0], np->cdata.fr, sizeof(np->cdata.fr));  in nvmft_port_create()
852 	port = &np->port;  in nvmft_port_create()
854 	port->frontend = &nvmft_frontend;  in nvmft_port_create()
855 	port->port_type = CTL_PORT_NVMF;  in nvmft_port_create()
856 	port->num_requested_ctl_io = max_io_qsize;  in nvmft_port_create()
857 	port->port_name = "nvmf";  in nvmft_port_create()
858 	port->physical_port = portid;  in nvmft_port_create()
859 	port->virtual_port = 0;  in nvmft_port_create()
860 	port->port_online = nvmft_online;  in nvmft_port_create()
861 	port->port_offline = nvmft_offline;  in nvmft_port_create()
862 	port->port_info = nvmft_info;  in nvmft_port_create()
863 	port->onoff_arg = np;  in nvmft_port_create()
864 	port->lun_enable = nvmft_lun_enable;  in nvmft_port_create()
865 	port->lun_disable = nvmft_lun_disable;  in nvmft_port_create()
866 	port->targ_lun_arg = np;  in nvmft_port_create()
867 	port->fe_datamove = nvmft_datamove;  in nvmft_port_create()
868 	port->fe_done = nvmft_done;  in nvmft_port_create()
869 	port->targ_port = -1;  in nvmft_port_create()
870 	port->options = nvlist_clone(req->args_nvl);  in nvmft_port_create()
875 		nvlist_destroy(port->options);  in nvmft_port_create()
877 		req->status = CTL_LUN_ERROR;  in nvmft_port_create()
878 		snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_create()
886 	req->status = CTL_LUN_OK;  in nvmft_port_create()
887 	req->result_nvl = nvlist_create(0);  in nvmft_port_create()
888 	nvlist_add_number(req->result_nvl, "port_id", port->targ_port);  in nvmft_port_create()
899 	 * ctladm port -r just provides the port_id, so permit looking  in nvmft_port_remove()
903 	subnqn = dnvlist_get_string(req->args_nvl, "subnqn", NULL);  in nvmft_port_remove()
905 		if (!nvlist_exists_string(req->args_nvl, "port_id")) {  in nvmft_port_remove()
906 			req->status = CTL_LUN_ERROR;  in nvmft_port_remove()
907 			snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_remove()
911 		if (!dnvlist_get_strnum(req->args_nvl, "port_id", ULONG_MAX,  in nvmft_port_remove()
913 			req->status = CTL_LUN_ERROR;  in nvmft_port_remove()
914 			snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_remove()
919 		if (nvlist_exists_string(req->args_nvl, "port_id")) {  in nvmft_port_remove()
920 			req->status = CTL_LUN_ERROR;  in nvmft_port_remove()
921 			snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_remove()
932 			req->status = CTL_LUN_ERROR;  in nvmft_port_remove()
933 			snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_remove()
941 			req->status = CTL_LUN_ERROR;  in nvmft_port_remove()
942 			snprintf(req->error_str, sizeof(req->error_str),  in nvmft_port_remove()
952 	mtx_lock(&np->lock);  in nvmft_port_remove()
953 	if (np->online) {  in nvmft_port_remove()
954 		mtx_unlock(&np->lock);  in nvmft_port_remove()
955 		ctl_port_offline(&np->port);  in nvmft_port_remove()
957 		mtx_unlock(&np->lock);  in nvmft_port_remove()
960 	req->status = CTL_LUN_OK;  in nvmft_port_remove()
976 	error = nvmf_unpack_ioc_nvlist(&cn->data.handoff, &nvl);  in nvmft_handoff()
978 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
979 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
988 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
989 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
996 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
997 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1004 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
1005 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1012 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
1013 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1018 	if (!nvmf_nqn_valid(data->subnqn)) {  in nvmft_handoff()
1019 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
1020 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1026 	np = nvmft_port_find(data->subnqn);  in nvmft_handoff()
1029 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
1030 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1034 	if (!np->online) {  in nvmft_handoff()
1036 		cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
1037 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1050 			cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
1051 			snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1058 			cn->status = CTL_NVMF_ERROR;  in nvmft_handoff()
1059 			snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_handoff()
1065 	cn->status = CTL_NVMF_OK;  in nvmft_handoff()
1081 	lp = &cn->data.list;  in nvmft_list()
1083 	sb = sbuf_new(NULL, NULL, lp->alloc_len, SBUF_FIXEDLEN |  in nvmft_list()
1086 		cn->status = CTL_NVMF_ERROR;  in nvmft_list()
1087 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_list()
1095 		mtx_lock(&np->lock);  in nvmft_list()
1096 		TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_list()
1102 			    ctrlr->cntlid,  in nvmft_list()
1103 			    ctrlr->hostnqn,  in nvmft_list()
1104 			    np->cdata.subnqn,  in nvmft_list()
1105 			    ctrlr->trtype);  in nvmft_list()
1107 		mtx_unlock(&np->lock);  in nvmft_list()
1113 		cn->status = CTL_NVMF_LIST_NEED_MORE_SPACE;  in nvmft_list()
1114 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_list()
1115 		    "Out of space, %d bytes is too small", lp->alloc_len);  in nvmft_list()
1119 	error = copyout(sbuf_data(sb), lp->conn_xml, sbuf_len(sb));  in nvmft_list()
1122 		cn->status = CTL_NVMF_ERROR;  in nvmft_list()
1123 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_list()
1127 	lp->fill_len = sbuf_len(sb);  in nvmft_list()
1128 	cn->status = CTL_NVMF_OK;  in nvmft_list()
1140 	tp = &cn->data.terminate;  in nvmft_terminate()
1145 		mtx_lock(&np->lock);  in nvmft_terminate()
1146 		TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_terminate()
1147 			if (tp->all != 0)  in nvmft_terminate()
1149 			else if (tp->cntlid != -1)  in nvmft_terminate()
1150 				match = tp->cntlid == ctrlr->cntlid;  in nvmft_terminate()
1151 			else if (tp->hostnqn[0] != '\0')  in nvmft_terminate()
1152 				match = strncmp(tp->hostnqn, ctrlr->hostnqn,  in nvmft_terminate()
1153 				    sizeof(tp->hostnqn)) == 0;  in nvmft_terminate()
1163 		mtx_unlock(&np->lock);  in nvmft_terminate()
1168 		cn->status = CTL_NVMF_ASSOCIATION_NOT_FOUND;  in nvmft_terminate()
1169 		snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_terminate()
1173 	cn->status = CTL_NVMF_OK;  in nvmft_terminate()
1186 		switch (req->reqtype) {  in nvmft_ioctl()
1194 			req->status = CTL_LUN_ERROR;  in nvmft_ioctl()
1195 			snprintf(req->error_str, sizeof(req->error_str),  in nvmft_ioctl()
1196 			    "Unsupported request type %d", req->reqtype);  in nvmft_ioctl()
1202 		switch (cn->type) {  in nvmft_ioctl()
1213 			cn->status = CTL_NVMF_ERROR;  in nvmft_ioctl()
1214 			snprintf(cn->error_str, sizeof(cn->error_str),  in nvmft_ioctl()
1215 			    "Invalid NVMeoF request type %d", cn->type);  in nvmft_ioctl()