210 nvme_qpair_complete_tracker(struct nvme_tracker *tr,  in nvme_qpair_complete_tracker()  argument
213 	struct nvme_qpair	*qpair = tr->qpair;  in nvme_qpair_complete_tracker()
219 	req = tr->req;  in nvme_qpair_complete_tracker()
241 			    tr->payload_dma_map,  in nvme_qpair_complete_tracker()
252 		nvme_qpair_submit_tracker(qpair, tr);  in nvme_qpair_complete_tracker()
256 			    tr->payload_dma_map);  in nvme_qpair_complete_tracker()
260 		tr->req = NULL;  in nvme_qpair_complete_tracker()
262 		TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq);  in nvme_qpair_complete_tracker()
263 		TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);  in nvme_qpair_complete_tracker()
297     struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,  in nvme_qpair_manual_complete_tracker()  argument
301 	struct nvme_qpair * qpair = tr->qpair;  in nvme_qpair_manual_complete_tracker()
308 	cpl.cid = tr->cid;  in nvme_qpair_manual_complete_tracker()
310 	nvme_qpair_complete_tracker(tr, &cpl, print_on_error);  in nvme_qpair_manual_complete_tracker()
341 	struct nvme_tracker	*tr;  in _nvme_qpair_process_completions()  local
437 			tr = qpair->act_tr[cpl.cid];  in _nvme_qpair_process_completions()
439 			tr = NULL;  in _nvme_qpair_process_completions()
442 		if (tr != NULL) {  in _nvme_qpair_process_completions()
443 			nvme_qpair_complete_tracker(tr, &cpl, ERROR_PRINT_ALL);  in _nvme_qpair_process_completions()
522 	struct nvme_tracker	*tr;  in nvme_qpair_construct()  local
638 		tr = malloc_domainset(sizeof(*tr), M_NVME,  in nvme_qpair_construct()
641 		    &tr->payload_dma_map);  in nvme_qpair_construct()
642 		tr->cid = i;  in nvme_qpair_construct()
643 		tr->qpair = qpair;  in nvme_qpair_construct()
644 		tr->prp = (uint64_t *)prp_list;  in nvme_qpair_construct()
645 		tr->prp_bus_addr = list_phys;  in nvme_qpair_construct()
646 		TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);  in nvme_qpair_construct()
698 	struct nvme_tracker	*tr;  in nvme_qpair_destroy()  local
716 		tr = TAILQ_FIRST(&qpair->free_tr);  in nvme_qpair_destroy()
717 		TAILQ_REMOVE(&qpair->free_tr, tr, tailq);  in nvme_qpair_destroy()
719 		    tr->payload_dma_map);  in nvme_qpair_destroy()
720 		free(tr, M_NVME);  in nvme_qpair_destroy()
755 	struct nvme_tracker	*tr;  in nvme_admin_qpair_abort_aers()  local
768 	tr = TAILQ_FIRST(&qpair->outstanding_tr);  in nvme_admin_qpair_abort_aers()
769 	while (tr != NULL) {  in nvme_admin_qpair_abort_aers()
770 		if (tr->req->cmd.opc != NVME_OPC_ASYNC_EVENT_REQUEST) {  in nvme_admin_qpair_abort_aers()
771 			tr = TAILQ_NEXT(tr, tailq);  in nvme_admin_qpair_abort_aers()
775 		nvme_qpair_manual_complete_tracker(tr,  in nvme_admin_qpair_abort_aers()
779 		tr = TAILQ_FIRST(&qpair->outstanding_tr);  in nvme_admin_qpair_abort_aers()
803 	struct nvme_tracker     *tr = arg;  in nvme_abort_complete()  local
814 	mtx_lock(&tr->qpair->lock);  in nvme_abort_complete()
815 	if ((status->cdw0 & 1) == 1 && tr->qpair->act_tr[tr->cid] != NULL) {  in nvme_abort_complete()
822 		nvme_printf(tr->qpair->ctrlr,  in nvme_abort_complete()
824 		nvme_qpair_manual_complete_tracker(tr,  in nvme_abort_complete()
834 	mtx_unlock(&tr->qpair->lock);  in nvme_abort_complete()
842 	struct nvme_tracker	*tr;  in nvme_qpair_timeout()  local
922 		TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) {  in nvme_qpair_timeout()
927 			if (tr->deadline == SBT_MAX)  in nvme_qpair_timeout()
935 			if (now <= tr->deadline)  in nvme_qpair_timeout()
961 		TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) {  in nvme_qpair_timeout()
966 			if (tr->deadline == SBT_MAX)  in nvme_qpair_timeout()
977 			if (now <= tr->deadline)  in nvme_qpair_timeout()
984 			    tr->req->cb_fn != nvme_abort_complete) {  in nvme_qpair_timeout()
991 				nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id,  in nvme_qpair_timeout()
992 				    nvme_abort_complete, tr);  in nvme_qpair_timeout()
1035 nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)  in nvme_qpair_submit_tracker()  argument
1043 	req = tr->req;  in nvme_qpair_submit_tracker()
1044 	req->cmd.cid = tr->cid;  in nvme_qpair_submit_tracker()
1045 	qpair->act_tr[tr->cid] = tr;  in nvme_qpair_submit_tracker()
1055 		tr->deadline = getsbinuptime() + timeout * SBT_1S;  in nvme_qpair_submit_tracker()
1062 		tr->deadline = SBT_MAX;  in nvme_qpair_submit_tracker()
1080 	struct nvme_tracker 	*tr = arg;  in nvme_payload_map()  local
1089 		nvme_printf(tr->qpair->ctrlr,  in nvme_payload_map()
1099 	tr->req->cmd.prp1 = htole64(seg[0].ds_addr);  in nvme_payload_map()
1102 		tr->req->cmd.prp2 = htole64(seg[1].ds_addr);  in nvme_payload_map()
1105 		tr->req->cmd.prp2 = htole64((uint64_t)tr->prp_bus_addr);  in nvme_payload_map()
1107 			tr->prp[cur_nseg-1] =  in nvme_payload_map()
1117 		tr->req->cmd.prp2 = 0;  in nvme_payload_map()
1120 	bus_dmamap_sync(tr->qpair->dma_tag_payload, tr->payload_dma_map,  in nvme_payload_map()
1122 	nvme_qpair_submit_tracker(tr->qpair, tr);  in nvme_payload_map()
1128 	struct nvme_tracker	*tr;  in _nvme_qpair_submit_request()  local
1134 	tr = TAILQ_FIRST(&qpair->free_tr);  in _nvme_qpair_submit_request()
1163 	if (tr == NULL || qpair->recovery_state != RECOVERY_NONE) {  in _nvme_qpair_submit_request()
1168 	TAILQ_REMOVE(&qpair->free_tr, tr, tailq);  in _nvme_qpair_submit_request()
1169 	TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);  in _nvme_qpair_submit_request()
1170 	tr->deadline = SBT_MAX;  in _nvme_qpair_submit_request()
1171 	tr->req = req;  in _nvme_qpair_submit_request()
1174 		nvme_qpair_submit_tracker(tr->qpair, tr);  in _nvme_qpair_submit_request()
1183 	err = bus_dmamap_load_mem(tr->qpair->dma_tag_payload,  in _nvme_qpair_submit_request()
1184 	    tr->payload_dma_map, &req->payload, nvme_payload_map, tr, 0);  in _nvme_qpair_submit_request()
1196 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in _nvme_qpair_submit_request()
1250 	struct nvme_tracker		*tr;  in nvme_admin_qpair_enable()  local
1264 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {  in nvme_admin_qpair_enable()
1265 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in nvme_admin_qpair_enable()
1283 	struct nvme_tracker		*tr;  in nvme_io_qpair_enable()  local
1296 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {  in nvme_io_qpair_enable()
1297 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in nvme_io_qpair_enable()
1329 	struct nvme_tracker	*tr, *tr_temp;  in nvme_qpair_disable()  local
1337 	TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {  in nvme_qpair_disable()
1338 		tr->deadline = SBT_MAX;  in nvme_qpair_disable()
1371 	struct nvme_tracker		*tr;  in nvme_qpair_fail()  local
1396 		tr = TAILQ_FIRST(&qpair->outstanding_tr);  in nvme_qpair_fail()
1402 		nvme_qpair_manual_complete_tracker(tr, NVME_SCT_GENERIC,  in nvme_qpair_fail()