19 static void ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,
53 	struct ufshci_req_queue *req_queue;  in ufshci_utmr_req_queue_construct()  local
60 	req_queue = &ctrlr->task_mgmt_req_queue;  in ufshci_utmr_req_queue_construct()
61 	req_queue->queue_mode = UFSHCI_Q_MODE_SDB;  in ufshci_utmr_req_queue_construct()
62 	req_queue->qops = sdb_utmr_qops;  in ufshci_utmr_req_queue_construct()
64 	error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES,  in ufshci_utmr_req_queue_construct()
87 	struct ufshci_req_queue *req_queue;  in ufshci_utr_req_queue_construct()  local
95 	req_queue = &ctrlr->transfer_req_queue;  in ufshci_utr_req_queue_construct()
96 	req_queue->queue_mode = UFSHCI_Q_MODE_SDB;  in ufshci_utr_req_queue_construct()
97 	req_queue->qops = sdb_utr_qops;  in ufshci_utr_req_queue_construct()
99 	error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES,  in ufshci_utr_req_queue_construct()
120 ufshci_req_queue_response_is_error(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_response_is_error()  argument
127 		ufshci_printf(req_queue->ctrlr, "Invalid OCS = 0x%x\n", ocs);  in ufshci_req_queue_response_is_error()
133 		ufshci_printf(req_queue->ctrlr,  in ufshci_req_queue_response_is_error()
161 ufshci_req_queue_manual_complete_request(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_manual_complete_request()  argument
169 	error = ufshci_req_queue_response_is_error(req_queue, ocs,  in ufshci_req_queue_manual_complete_request()
173 		ufshci_printf(req_queue->ctrlr,  in ufshci_req_queue_manual_complete_request()
187 	struct ufshci_req_queue *req_queue;  in ufshci_req_queue_fail()  local
197 	req_queue = &ctrlr->transfer_req_queue;  in ufshci_req_queue_fail()
199 	for (i = 0; i < req_queue->num_entries; i++) {  in ufshci_req_queue_fail()
205 			ufshci_req_queue_manual_complete_request(req_queue, req,  in ufshci_req_queue_fail()
228 	struct ufshci_req_queue *req_queue = tr->req_queue;  in ufshci_req_queue_complete_tracker()  local
238 	if (req_queue->is_task_mgmt) {  in ufshci_req_queue_complete_tracker()
246 		bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,  in ufshci_req_queue_complete_tracker()
256 	error = ufshci_req_queue_response_is_error(req_queue, ocs,  in ufshci_req_queue_complete_tracker()
263 	    req->retries < req_queue->ctrlr->retry_count;  in ufshci_req_queue_complete_tracker()
266 	if (error && req->retries >= req_queue->ctrlr->retry_count && retriable)  in ufshci_req_queue_complete_tracker()
276 			bus_dmamap_sync(req_queue->dma_tag_payload,  in ufshci_req_queue_complete_tracker()
288 	req_queue->qops.clear_cpl_ntf(req_queue->ctrlr, tr);  in ufshci_req_queue_complete_tracker()
292 		ufshci_req_queue_submit_tracker(req_queue, tr,  in ufshci_req_queue_complete_tracker()
296 			bus_dmamap_unload(req_queue->dma_tag_payload,  in ufshci_req_queue_complete_tracker()
310 ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue)  in ufshci_req_queue_process_completions()  argument
312 	return (req_queue->qops.process_cpl(req_queue));  in ufshci_req_queue_process_completions()
328 		ufshci_printf(tr->req_queue->ctrlr,  in ufshci_payload_map()
347 	bus_dmamap_sync(tr->req_queue->dma_tag_payload, tr->payload_dma_map,  in ufshci_payload_map()
363 	error = bus_dmamap_load_mem(tr->req_queue->dma_tag_payload,  in ufshci_req_queue_prepare_prdt()
374 		ufshci_printf(tr->req_queue->ctrlr,  in ufshci_req_queue_prepare_prdt()
434 ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_submit_tracker()  argument
437 	struct ufshci_controller *ctrlr = req_queue->ctrlr;  in ufshci_req_queue_submit_tracker()
443 	mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED);  in ufshci_req_queue_submit_tracker()
447 	if (req_queue->is_task_mgmt) {  in ufshci_req_queue_submit_tracker()
470 		bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,  in ufshci_req_queue_submit_tracker()
480 	req_queue->qops.ring_doorbell(ctrlr, tr);  in ufshci_req_queue_submit_tracker()
484 _ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,  in _ufshci_req_queue_submit_request()  argument
490 	mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED);  in _ufshci_req_queue_submit_request()
492 	error = req_queue->qops.reserve_slot(req_queue, &tr);  in _ufshci_req_queue_submit_request()
494 		ufshci_printf(req_queue->ctrlr, "Failed to get tracker");  in _ufshci_req_queue_submit_request()
511 	ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction);  in _ufshci_req_queue_submit_request()
517 ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_submit_request()  argument
525 	hwq = req_queue->qops.get_hw_queue(req_queue);  in ufshci_req_queue_submit_request()
529 	error = _ufshci_req_queue_submit_request(req_queue, req);  in ufshci_req_queue_submit_request()