28 	struct nvmf_qpair *qp;  member
51 nvmf_allocate_request(struct nvmf_host_qpair *qp, void *sqe,  in nvmf_allocate_request()  argument
64 	mtx_lock(&qp->lock);  in nvmf_allocate_request()
65 	nq = qp->qp;  in nvmf_allocate_request()
67 		mtx_unlock(&qp->lock);  in nvmf_allocate_request()
71 	qp->allocating++;  in nvmf_allocate_request()
72 	MPASS(qp->allocating != 0);  in nvmf_allocate_request()
73 	mtx_unlock(&qp->lock);  in nvmf_allocate_request()
75 	req->qp = qp;  in nvmf_allocate_request()
84 	mtx_lock(&qp->lock);  in nvmf_allocate_request()
85 	qp->allocating--;  in nvmf_allocate_request()
86 	if (qp->allocating == 0 && qp->shutting_down)  in nvmf_allocate_request()
87 		wakeup(qp);  in nvmf_allocate_request()
88 	mtx_unlock(&qp->lock);  in nvmf_allocate_request()
114 nvmf_dispatch_command(struct nvmf_host_qpair *qp, struct nvmf_host_command *cmd)  in nvmf_dispatch_command()  argument
116 	struct nvmf_softc *sc = qp->sc;  in nvmf_dispatch_command()
122 	mtx_assert(&qp->lock, MA_OWNED);  in nvmf_dispatch_command()
124 	qp->submitted++;  in nvmf_dispatch_command()
131 	new_sqtail = (qp->sqtail + 1) % (qp->num_commands + 1);  in nvmf_dispatch_command()
132 	KASSERT(new_sqtail != qp->sqhd, ("%s: qp %p is full", __func__, qp));  in nvmf_dispatch_command()
133 	qp->sqtail = new_sqtail;  in nvmf_dispatch_command()
134 	mtx_unlock(&qp->lock);  in nvmf_dispatch_command()
160 	struct nvmf_host_qpair *qp = arg;  in nvmf_qp_error()  local
161 	struct nvmf_softc *sc = qp->sc;  in nvmf_qp_error()
166 		    qp->name);  in nvmf_qp_error()
173 	struct nvmf_host_qpair *qp = arg;  in nvmf_receive_capsule()  local
174 	struct nvmf_softc *sc = qp->sc;  in nvmf_receive_capsule()
191 	if (cid > qp->num_commands) {  in nvmf_receive_capsule()
200 	mtx_lock(&qp->lock);  in nvmf_receive_capsule()
201 	if (qp->sq_flow_control) {  in nvmf_receive_capsule()
203 			qp->sqhd = le16toh(cqe->sqhd);  in nvmf_receive_capsule()
209 		qp->sqhd = (qp->sqhd + 1) % (qp->num_commands + 1);  in nvmf_receive_capsule()
216 	if (qp->qp == NULL) {  in nvmf_receive_capsule()
219 		    qp->name);  in nvmf_receive_capsule()
220 		mtx_unlock(&qp->lock);  in nvmf_receive_capsule()
225 	cmd = qp->active_commands[cid];  in nvmf_receive_capsule()
227 		mtx_unlock(&qp->lock);  in nvmf_receive_capsule()
239 	if (STAILQ_EMPTY(&qp->pending_requests)) {  in nvmf_receive_capsule()
240 		qp->active_commands[cid] = NULL;  in nvmf_receive_capsule()
241 		TAILQ_INSERT_TAIL(&qp->free_commands, cmd, link);  in nvmf_receive_capsule()
242 		mtx_unlock(&qp->lock);  in nvmf_receive_capsule()
244 		cmd->req = STAILQ_FIRST(&qp->pending_requests);  in nvmf_receive_capsule()
245 		STAILQ_REMOVE_HEAD(&qp->pending_requests, link);  in nvmf_receive_capsule()
246 		nvmf_dispatch_command(qp, cmd);  in nvmf_receive_capsule()
255 nvmf_sysctls_qp(struct nvmf_softc *sc, struct nvmf_host_qpair *qp,  in nvmf_sysctls_qp()  argument
258 	struct sysctl_ctx_list *ctx = &qp->sysctl_ctx;  in nvmf_sysctls_qp()
275 	    NULL, qp->num_commands + 1, "Number of entries in queue");  in nvmf_sysctls_qp()
276 	SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_head", CTLFLAG_RD, &qp->sqhd,  in nvmf_sysctls_qp()
278 	SYSCTL_ADD_U16(ctx, list, OID_AUTO, "sq_tail", CTLFLAG_RD, &qp->sqtail,  in nvmf_sysctls_qp()
281 	    &qp->submitted, 0, "Number of commands submitted");  in nvmf_sysctls_qp()
289 	struct nvmf_host_qpair *qp;  in nvmf_init_qp()  local
294 	qp = malloc(sizeof(*qp), M_NVMF, M_WAITOK | M_ZERO);  in nvmf_init_qp()
295 	qp->sc = sc;  in nvmf_init_qp()
296 	qp->sq_flow_control = nvlist_get_bool(nvl, "sq_flow_control");  in nvmf_init_qp()
297 	qp->sqhd = nvlist_get_number(nvl, "sqhd");  in nvmf_init_qp()
298 	qp->sqtail = nvlist_get_number(nvl, "sqtail");  in nvmf_init_qp()
299 	strlcpy(qp->name, name, sizeof(qp->name));  in nvmf_init_qp()
300 	mtx_init(&qp->lock, "nvmf qp", NULL, MTX_DEF);  in nvmf_init_qp()
301 	(void)sysctl_ctx_init(&qp->sysctl_ctx);  in nvmf_init_qp()
307 	qp->num_commands = nvlist_get_number(nvl, "qsize") - 1;  in nvmf_init_qp()
309 		qp->num_commands += sc->num_aer;  in nvmf_init_qp()
311 	qp->active_commands = malloc(sizeof(*qp->active_commands) *  in nvmf_init_qp()
312 	    qp->num_commands, M_NVMF, M_WAITOK | M_ZERO);  in nvmf_init_qp()
313 	TAILQ_INIT(&qp->free_commands);  in nvmf_init_qp()
314 	for (i = 0; i < qp->num_commands; i++) {  in nvmf_init_qp()
317 		TAILQ_INSERT_TAIL(&qp->free_commands, cmd, link);  in nvmf_init_qp()
319 	STAILQ_INIT(&qp->pending_requests);  in nvmf_init_qp()
321 	qp->qp = nvmf_allocate_qpair(trtype, false, nvl, nvmf_qp_error, qp,  in nvmf_init_qp()
322 	    nvmf_receive_capsule, qp);  in nvmf_init_qp()
323 	if (qp->qp == NULL) {  in nvmf_init_qp()
324 		(void)sysctl_ctx_free(&qp->sysctl_ctx);  in nvmf_init_qp()
325 		TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) {  in nvmf_init_qp()
326 			TAILQ_REMOVE(&qp->free_commands, cmd, link);  in nvmf_init_qp()
329 		free(qp->active_commands, M_NVMF);  in nvmf_init_qp()
330 		mtx_destroy(&qp->lock);  in nvmf_init_qp()
331 		free(qp, M_NVMF);  in nvmf_init_qp()
335 	nvmf_sysctls_qp(sc, qp, admin, qid);  in nvmf_init_qp()
337 	return (qp);  in nvmf_init_qp()
341 nvmf_shutdown_qp(struct nvmf_host_qpair *qp)  in nvmf_shutdown_qp()  argument
347 	mtx_lock(&qp->lock);  in nvmf_shutdown_qp()
348 	nq = qp->qp;  in nvmf_shutdown_qp()
349 	qp->qp = NULL;  in nvmf_shutdown_qp()
352 		while (qp->shutting_down)  in nvmf_shutdown_qp()
353 			mtx_sleep(qp, &qp->lock, 0, "nvmfqpsh", 0);  in nvmf_shutdown_qp()
354 		mtx_unlock(&qp->lock);  in nvmf_shutdown_qp()
357 	qp->shutting_down = true;  in nvmf_shutdown_qp()
358 	while (qp->allocating != 0)  in nvmf_shutdown_qp()
359 		mtx_sleep(qp, &qp->lock, 0, "nvmfqpqu", 0);  in nvmf_shutdown_qp()
360 	mtx_unlock(&qp->lock);  in nvmf_shutdown_qp()
371 	for (u_int i = 0; i < qp->num_commands; i++) {  in nvmf_shutdown_qp()
372 		cmd = qp->active_commands[i];  in nvmf_shutdown_qp()
385 	while (!STAILQ_EMPTY(&qp->pending_requests)) {  in nvmf_shutdown_qp()
386 		req = STAILQ_FIRST(&qp->pending_requests);  in nvmf_shutdown_qp()
387 		STAILQ_REMOVE_HEAD(&qp->pending_requests, link);  in nvmf_shutdown_qp()
396 	mtx_lock(&qp->lock);  in nvmf_shutdown_qp()
397 	qp->shutting_down = false;  in nvmf_shutdown_qp()
398 	mtx_unlock(&qp->lock);  in nvmf_shutdown_qp()
399 	wakeup(qp);  in nvmf_shutdown_qp()
403 nvmf_destroy_qp(struct nvmf_host_qpair *qp)  in nvmf_destroy_qp()  argument
407 	nvmf_shutdown_qp(qp);  in nvmf_destroy_qp()
408 	(void)sysctl_ctx_free(&qp->sysctl_ctx);  in nvmf_destroy_qp()
410 	TAILQ_FOREACH_SAFE(cmd, &qp->free_commands, link, ncmd) {  in nvmf_destroy_qp()
411 		TAILQ_REMOVE(&qp->free_commands, cmd, link);  in nvmf_destroy_qp()
414 	free(qp->active_commands, M_NVMF);  in nvmf_destroy_qp()
415 	mtx_destroy(&qp->lock);  in nvmf_destroy_qp()
416 	free(qp, M_NVMF);  in nvmf_destroy_qp()
422 	struct nvmf_host_qpair *qp;  in nvmf_submit_request()  local
425 	qp = req->qp;  in nvmf_submit_request()
426 	mtx_lock(&qp->lock);  in nvmf_submit_request()
427 	if (qp->qp == NULL) {  in nvmf_submit_request()
428 		mtx_unlock(&qp->lock);  in nvmf_submit_request()
435 	cmd = TAILQ_FIRST(&qp->free_commands);  in nvmf_submit_request()
441 		STAILQ_INSERT_TAIL(&qp->pending_requests, req, link);  in nvmf_submit_request()
442 		mtx_unlock(&qp->lock);  in nvmf_submit_request()
446 	TAILQ_REMOVE(&qp->free_commands, cmd, link);  in nvmf_submit_request()
447 	KASSERT(qp->active_commands[cmd->cid] == NULL,  in nvmf_submit_request()
449 	qp->active_commands[cmd->cid] = cmd;  in nvmf_submit_request()
451 	nvmf_dispatch_command(qp, cmd);  in nvmf_submit_request()