414 nvme_qpair_complete_tracker(struct nvme_tracker *tr,  in nvme_qpair_complete_tracker()  argument
417 	struct nvme_qpair	*qpair = tr->qpair;  in nvme_qpair_complete_tracker()
423 	req = tr->req;  in nvme_qpair_complete_tracker()
445 			    tr->payload_dma_map,  in nvme_qpair_complete_tracker()
456 		nvme_qpair_submit_tracker(qpair, tr);  in nvme_qpair_complete_tracker()
460 			    tr->payload_dma_map);  in nvme_qpair_complete_tracker()
464 		tr->req = NULL;  in nvme_qpair_complete_tracker()
466 		TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq);  in nvme_qpair_complete_tracker()
467 		TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);  in nvme_qpair_complete_tracker()
501     struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,  in nvme_qpair_manual_complete_tracker()  argument
505 	struct nvme_qpair * qpair = tr->qpair;  in nvme_qpair_manual_complete_tracker()
512 	cpl.cid = tr->cid;  in nvme_qpair_manual_complete_tracker()
514 	nvme_qpair_complete_tracker(tr, &cpl, print_on_error);  in nvme_qpair_manual_complete_tracker()
545 	struct nvme_tracker	*tr;  in _nvme_qpair_process_completions()  local
641 			tr = qpair->act_tr[cpl.cid];  in _nvme_qpair_process_completions()
643 			tr = NULL;  in _nvme_qpair_process_completions()
646 		if (tr != NULL) {  in _nvme_qpair_process_completions()
647 			nvme_qpair_complete_tracker(tr, &cpl, ERROR_PRINT_ALL);  in _nvme_qpair_process_completions()
726 	struct nvme_tracker	*tr;  in nvme_qpair_construct()  local
842 		tr = malloc_domainset(sizeof(*tr), M_NVME,  in nvme_qpair_construct()
845 		    &tr->payload_dma_map);  in nvme_qpair_construct()
846 		tr->cid = i;  in nvme_qpair_construct()
847 		tr->qpair = qpair;  in nvme_qpair_construct()
848 		tr->prp = (uint64_t *)prp_list;  in nvme_qpair_construct()
849 		tr->prp_bus_addr = list_phys;  in nvme_qpair_construct()
850 		TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);  in nvme_qpair_construct()
902 	struct nvme_tracker	*tr;  in nvme_qpair_destroy()  local
920 		tr = TAILQ_FIRST(&qpair->free_tr);  in nvme_qpair_destroy()
921 		TAILQ_REMOVE(&qpair->free_tr, tr, tailq);  in nvme_qpair_destroy()
923 		    tr->payload_dma_map);  in nvme_qpair_destroy()
924 		free(tr, M_NVME);  in nvme_qpair_destroy()
959 	struct nvme_tracker	*tr;  in nvme_admin_qpair_abort_aers()  local
972 	tr = TAILQ_FIRST(&qpair->outstanding_tr);  in nvme_admin_qpair_abort_aers()
973 	while (tr != NULL) {  in nvme_admin_qpair_abort_aers()
974 		if (tr->req->cmd.opc != NVME_OPC_ASYNC_EVENT_REQUEST) {  in nvme_admin_qpair_abort_aers()
975 			tr = TAILQ_NEXT(tr, tailq);  in nvme_admin_qpair_abort_aers()
979 		nvme_qpair_manual_complete_tracker(tr,  in nvme_admin_qpair_abort_aers()
983 		tr = TAILQ_FIRST(&qpair->outstanding_tr);  in nvme_admin_qpair_abort_aers()
1007 	struct nvme_tracker     *tr = arg;  in nvme_abort_complete()  local
1018 	mtx_lock(&tr->qpair->lock);  in nvme_abort_complete()
1019 	if ((status->cdw0 & 1) == 1 && tr->qpair->act_tr[tr->cid] != NULL) {  in nvme_abort_complete()
1026 		nvme_printf(tr->qpair->ctrlr,  in nvme_abort_complete()
1028 		nvme_qpair_manual_complete_tracker(tr,  in nvme_abort_complete()
1038 	mtx_unlock(&tr->qpair->lock);  in nvme_abort_complete()
1046 	struct nvme_tracker	*tr;  in nvme_qpair_timeout()  local
1126 		TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) {  in nvme_qpair_timeout()
1131 			if (tr->deadline == SBT_MAX)  in nvme_qpair_timeout()
1139 			if (now <= tr->deadline)  in nvme_qpair_timeout()
1165 		TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) {  in nvme_qpair_timeout()
1170 			if (tr->deadline == SBT_MAX)  in nvme_qpair_timeout()
1181 			if (now <= tr->deadline)  in nvme_qpair_timeout()
1188 			    tr->req->cb_fn != nvme_abort_complete) {  in nvme_qpair_timeout()
1195 				nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id,  in nvme_qpair_timeout()
1196 				    nvme_abort_complete, tr);  in nvme_qpair_timeout()
1239 nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)  in nvme_qpair_submit_tracker()  argument
1247 	req = tr->req;  in nvme_qpair_submit_tracker()
1248 	req->cmd.cid = tr->cid;  in nvme_qpair_submit_tracker()
1249 	qpair->act_tr[tr->cid] = tr;  in nvme_qpair_submit_tracker()
1259 		tr->deadline = getsbinuptime() + timeout * SBT_1S;  in nvme_qpair_submit_tracker()
1266 		tr->deadline = SBT_MAX;  in nvme_qpair_submit_tracker()
1284 	struct nvme_tracker 	*tr = arg;  in nvme_payload_map()  local
1293 		nvme_printf(tr->qpair->ctrlr,  in nvme_payload_map()
1303 	tr->req->cmd.prp1 = htole64(seg[0].ds_addr);  in nvme_payload_map()
1306 		tr->req->cmd.prp2 = htole64(seg[1].ds_addr);  in nvme_payload_map()
1309 		tr->req->cmd.prp2 = htole64((uint64_t)tr->prp_bus_addr);  in nvme_payload_map()
1311 			tr->prp[cur_nseg-1] =  in nvme_payload_map()
1321 		tr->req->cmd.prp2 = 0;  in nvme_payload_map()
1324 	bus_dmamap_sync(tr->qpair->dma_tag_payload, tr->payload_dma_map,  in nvme_payload_map()
1326 	nvme_qpair_submit_tracker(tr->qpair, tr);  in nvme_payload_map()
1332 	struct nvme_tracker	*tr;  in _nvme_qpair_submit_request()  local
1338 	tr = TAILQ_FIRST(&qpair->free_tr);  in _nvme_qpair_submit_request()
1367 	if (tr == NULL || qpair->recovery_state != RECOVERY_NONE) {  in _nvme_qpair_submit_request()
1372 	TAILQ_REMOVE(&qpair->free_tr, tr, tailq);  in _nvme_qpair_submit_request()
1373 	TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);  in _nvme_qpair_submit_request()
1374 	tr->deadline = SBT_MAX;  in _nvme_qpair_submit_request()
1375 	tr->req = req;  in _nvme_qpair_submit_request()
1378 		nvme_qpair_submit_tracker(tr->qpair, tr);  in _nvme_qpair_submit_request()
1387 	err = bus_dmamap_load_mem(tr->qpair->dma_tag_payload,  in _nvme_qpair_submit_request()
1388 	    tr->payload_dma_map, &req->payload, nvme_payload_map, tr, 0);  in _nvme_qpair_submit_request()
1400 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in _nvme_qpair_submit_request()
1454 	struct nvme_tracker		*tr;  in nvme_admin_qpair_enable()  local
1468 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {  in nvme_admin_qpair_enable()
1469 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in nvme_admin_qpair_enable()
1487 	struct nvme_tracker		*tr;  in nvme_io_qpair_enable()  local
1500 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {  in nvme_io_qpair_enable()
1501 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in nvme_io_qpair_enable()
1533 	struct nvme_tracker	*tr, *tr_temp;  in nvme_qpair_disable()  local
1541 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {  in nvme_qpair_disable()
1542 		tr->deadline = SBT_MAX;  in nvme_qpair_disable()
1575 	struct nvme_tracker		*tr;  in nvme_qpair_fail()  local
1600 		tr = TAILQ_FIRST(&qpair->outstanding_tr);  in nvme_qpair_fail()
1606 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in nvme_qpair_fail()