1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
4  * Copyright (c) 2023-2024 Chelsio Communications, Inc.
11 #include <sys/lock.h>
39 	struct mtx lock;  member
52 	struct nvmft_controller *ctrlr = qp->ctrlr;  in nvmft_qpair_error()
63 		nvmft_printf(ctrlr, "error %d on %s\n", error, qp->name);  in nvmft_qpair_error()
71 	struct nvmft_controller *ctrlr = qp->ctrlr;  in nvmft_receive_capsule()
78 		    qp->name, le16toh(cmd->cid), cmd->opc);  in nvmft_receive_capsule()
90 	/* Don't bother byte-swapping CID. */  in nvmft_receive_capsule()
91 	if (BIT_TEST_SET_ATOMIC(NUM_CIDS, cmd->cid, qp->cids)) {  in nvmft_receive_capsule()
97 	if (qp->admin)  in nvmft_receive_capsule()
100 		nvmft_handle_io_command(qp, qp->qid, nc);  in nvmft_receive_capsule()
110 	qp->admin = nvlist_get_bool(params, "admin");  in nvmft_qpair_init()
111 	qp->sq_flow_control = nvlist_get_bool(params, "sq_flow_control");  in nvmft_qpair_init()
112 	qp->qsize = nvlist_get_number(params, "qsize");  in nvmft_qpair_init()
113 	qp->qid = qid;  in nvmft_qpair_init()
114 	qp->sqhd = nvlist_get_number(params, "sqhd");  in nvmft_qpair_init()
115 	strlcpy(qp->name, name, sizeof(qp->name));  in nvmft_qpair_init()
116 	mtx_init(&qp->lock, "nvmft qp", NULL, MTX_DEF);  in nvmft_qpair_init()
117 	qp->cids = BITSET_ALLOC(NUM_CIDS, M_NVMFT, M_WAITOK | M_ZERO);  in nvmft_qpair_init()
118 	STAILQ_INIT(&qp->datamove_queue);  in nvmft_qpair_init()
119 	TASK_INIT(&qp->datamove_task, 0, nvmft_datamove_task, qp);  in nvmft_qpair_init()
121 	qp->qp = nvmf_allocate_qpair(trtype, true, params, nvmft_qpair_error,  in nvmft_qpair_init()
123 	if (qp->qp == NULL) {  in nvmft_qpair_init()
124 		mtx_destroy(&qp->lock);  in nvmft_qpair_init()
125 		free(qp->cids, M_NVMFT);  in nvmft_qpair_init()
130 	refcount_init(&qp->qp_refs, 1);  in nvmft_qpair_init()
142 	mtx_lock(&qp->lock);  in nvmft_qpair_shutdown()
143 	nq = qp->qp;  in nvmft_qpair_shutdown()
144 	qp->qp = NULL;  in nvmft_qpair_shutdown()
145 	STAILQ_CONCAT(&datamove_queue, &qp->datamove_queue);  in nvmft_qpair_shutdown()
146 	mtx_unlock(&qp->lock);  in nvmft_qpair_shutdown()
147 	if (nq != NULL && refcount_release(&qp->qp_refs))  in nvmft_qpair_shutdown()
155 	nvmft_drain_task(&qp->datamove_task);  in nvmft_qpair_shutdown()
162 	mtx_destroy(&qp->lock);  in nvmft_qpair_destroy()
163 	free(qp->cids, M_NVMFT);  in nvmft_qpair_destroy()
170 	return (qp->ctrlr);  in nvmft_qpair_ctrlr()
176 	return (qp->qid);  in nvmft_qpair_id()
182 	return (qp->name);  in nvmft_qpair_name()
188 	return (nvmf_max_ioccsz(qp->qp));  in nvmft_max_ioccsz()
200 	mtx_lock(&qp->lock);  in _nvmft_send_response()
201 	nq = qp->qp;  in _nvmft_send_response()
203 		mtx_unlock(&qp->lock);  in _nvmft_send_response()
206 	refcount_acquire(&qp->qp_refs);  in _nvmft_send_response()
209 	if (qp->sq_flow_control) {  in _nvmft_send_response()
210 		qp->sqhd = (qp->sqhd + 1) % qp->qsize;  in _nvmft_send_response()
211 		cpl.sqhd = htole16(qp->sqhd);  in _nvmft_send_response()
214 	mtx_unlock(&qp->lock);  in _nvmft_send_response()
220 	if (refcount_release(&qp->qp_refs))  in _nvmft_send_response()
230 	/* Don't bother byte-swapping CID. */  in nvmft_command_completed()
231 	KASSERT(BIT_ISSET(NUM_CIDS, cmd->cid, qp->cids),  in nvmft_command_completed()
232 	    ("%s: CID %u not busy", __func__, cmd->cid));  in nvmft_command_completed()
234 	BIT_CLR_ATOMIC(NUM_CIDS, cmd->cid, qp->cids);  in nvmft_command_completed()
242 	/* Don't bother byte-swapping CID. */  in nvmft_send_response()
243 	KASSERT(BIT_ISSET(NUM_CIDS, cpl->cid, qp->cids),  in nvmft_send_response()
244 	    ("%s: CID %u not busy", __func__, cpl->cid));  in nvmft_send_response()
246 	BIT_CLR_ATOMIC(NUM_CIDS, cpl->cid, qp->cids);  in nvmft_send_response()
251 nvmft_init_cqe(void *cqe, struct nvmf_capsule *nc, uint16_t status)  in nvmft_init_cqe()  argument
257 	cpl->cid = cmd->cid;  in nvmft_init_cqe()
258 	cpl->status = htole16(status);  in nvmft_init_cqe()
266 	uint16_t status;  in nvmft_send_error()  local
268 	status = NVMEF(NVME_STATUS_SCT, sc_type) |  in nvmft_send_error()
270 	nvmft_init_cqe(&cpl, nc, status);  in nvmft_send_error()
282  * This version doesn't clear CID in qp->cids and is used for errors
290 	uint16_t status;  in _nvmft_send_generic_error()  local
292 	status = NVMEF(NVME_STATUS_SCT, NVME_SCT_GENERIC) |  in _nvmft_send_generic_error()
294 	nvmft_init_cqe(&cpl, nc, status);  in _nvmft_send_generic_error()
306     const struct nvmf_fabric_connect_cmd *cmd, uint16_t status)  in nvmft_init_connect_rsp()  argument
309 	rsp->cid = cmd->cid;  in nvmft_init_connect_rsp()
310 	rsp->status = htole16(status);  in nvmft_init_connect_rsp()
321 	mtx_lock(&qp->lock);  in nvmft_send_connect_response()
322 	nq = qp->qp;  in nvmft_send_connect_response()
324 		mtx_unlock(&qp->lock);  in nvmft_send_connect_response()
327 	refcount_acquire(&qp->qp_refs);  in nvmft_send_connect_response()
328 	mtx_unlock(&qp->lock);  in nvmft_send_connect_response()
330 	rc = nvmf_allocate_response(qp->qp, rsp, M_WAITOK);  in nvmft_send_connect_response()
334 	if (refcount_release(&qp->qp_refs))  in nvmft_send_connect_response()
345 	uint16_t status;  in nvmft_connect_error()  local
347 	status = NVMEF(NVME_STATUS_SCT, sc_type) |  in nvmft_connect_error()
349 	nvmft_init_connect_rsp(&rsp, cmd, status);  in nvmft_connect_error()
373 	qp->ctrlr = ctrlr;  in nvmft_finish_accept()
375 	if (qp->sq_flow_control)  in nvmft_finish_accept()
376 		rsp.sqhd = htole16(qp->sqhd);  in nvmft_finish_accept()
379 	rsp.status_code_specific.success.cntlid = htole16(ctrlr->cntlid);  in nvmft_finish_accept()
388 	mtx_lock(&qp->lock);  in nvmft_qpair_datamove()
389 	if (qp->qp == NULL) {  in nvmft_qpair_datamove()
390 		mtx_unlock(&qp->lock);  in nvmft_qpair_datamove()
394 	enqueue_task = STAILQ_EMPTY(&qp->datamove_queue);  in nvmft_qpair_datamove()
395 	STAILQ_INSERT_TAIL(&qp->datamove_queue, &io->io_hdr, links);  in nvmft_qpair_datamove()
396 	mtx_unlock(&qp->lock);  in nvmft_qpair_datamove()
398 		nvmft_enqueue_task(&qp->datamove_task);  in nvmft_qpair_datamove()
408 	mtx_lock(&qp->lock);  in nvmft_datamove_task()
409 	while (!STAILQ_EMPTY(&qp->datamove_queue)) {  in nvmft_datamove_task()
410 		io = (union ctl_io *)STAILQ_FIRST(&qp->datamove_queue);  in nvmft_datamove_task()
411 		STAILQ_REMOVE_HEAD(&qp->datamove_queue, links);  in nvmft_datamove_task()
412 		abort = (qp->qp == NULL);  in nvmft_datamove_task()
413 		mtx_unlock(&qp->lock);  in nvmft_datamove_task()
418 		mtx_lock(&qp->lock);  in nvmft_datamove_task()
420 	mtx_unlock(&qp->lock);  in nvmft_datamove_task()