Lines Matching +full:ufs +full:- +full:disable +full:- +full:mcq
1 /*-
5 * SPDX-License-Identifier: BSD-2-Clause
27 .disable = ufshci_req_sdb_disable,
42 .disable = ufshci_req_sdb_disable,
62 req_queue = &ctrlr->task_mgmt_req_queue; in ufshci_utmr_req_queue_construct()
63 req_queue->queue_mode = UFSHCI_Q_MODE_SDB; in ufshci_utmr_req_queue_construct()
64 req_queue->qops = sdb_utmr_qops; in ufshci_utmr_req_queue_construct()
66 error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES, in ufshci_utmr_req_queue_construct()
75 ctrlr->task_mgmt_req_queue.qops.destroy(ctrlr, in ufshci_utmr_req_queue_destroy()
76 &ctrlr->task_mgmt_req_queue); in ufshci_utmr_req_queue_destroy()
82 ctrlr->task_mgmt_req_queue.qops.disable(ctrlr, in ufshci_utmr_req_queue_disable()
83 &ctrlr->task_mgmt_req_queue); in ufshci_utmr_req_queue_disable()
89 return (ctrlr->task_mgmt_req_queue.qops.enable(ctrlr, in ufshci_utmr_req_queue_enable()
90 &ctrlr->task_mgmt_req_queue)); in ufshci_utmr_req_queue_enable()
100 * Currently, it does not support MCQ mode, so it should be set to SDB in ufshci_utr_req_queue_construct()
104 req_queue = &ctrlr->transfer_req_queue; in ufshci_utr_req_queue_construct()
105 req_queue->queue_mode = UFSHCI_Q_MODE_SDB; in ufshci_utr_req_queue_construct()
106 req_queue->qops = sdb_utr_qops; in ufshci_utr_req_queue_construct()
108 error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES, in ufshci_utr_req_queue_construct()
117 ctrlr->transfer_req_queue.qops.destroy(ctrlr, in ufshci_utr_req_queue_destroy()
118 &ctrlr->transfer_req_queue); in ufshci_utr_req_queue_destroy()
124 ctrlr->transfer_req_queue.qops.disable(ctrlr, in ufshci_utr_req_queue_disable()
125 &ctrlr->transfer_req_queue); in ufshci_utr_req_queue_disable()
131 return (ctrlr->transfer_req_queue.qops.enable(ctrlr, in ufshci_utr_req_queue_enable()
132 &ctrlr->transfer_req_queue)); in ufshci_utr_req_queue_enable()
143 ufshci_printf(req_queue->ctrlr, "Invalid OCS = 0x%x\n", ocs); in ufshci_req_queue_response_is_error()
148 if (response->header.response != UFSHCI_RESPONSE_CODE_TARGET_SUCCESS) { in ufshci_req_queue_response_is_error()
149 ufshci_printf(req_queue->ctrlr, in ufshci_req_queue_response_is_error()
151 response->header.ext_iid_or_function, in ufshci_req_queue_response_is_error()
152 response->header.response); in ufshci_req_queue_response_is_error()
166 mtx_assert(&tr->hwq->qlock, MA_NOTOWNED); in ufshci_req_queue_manual_complete_tracker()
168 resp_header = (struct ufshci_upiu_header *)tr->ucd->response_upiu; in ufshci_req_queue_manual_complete_tracker()
169 resp_header->response = rc; in ufshci_req_queue_manual_complete_tracker()
171 desc = &tr->hwq->utrd[tr->slot_num]; in ufshci_req_queue_manual_complete_tracker()
172 desc->overall_command_status = ocs; in ufshci_req_queue_manual_complete_tracker()
190 ufshci_printf(req_queue->ctrlr, in ufshci_req_queue_manual_complete_request()
194 if (req->cb_fn) in ufshci_req_queue_manual_complete_request()
195 req->cb_fn(req->cb_arg, &cpl, error); in ufshci_req_queue_manual_complete_request()
209 if (!mtx_initialized(&hwq->qlock)) in ufshci_req_queue_fail()
212 mtx_lock(&hwq->qlock); in ufshci_req_queue_fail()
214 req_queue = &ctrlr->transfer_req_queue; in ufshci_req_queue_fail()
216 for (i = 0; i < req_queue->num_entries; i++) { in ufshci_req_queue_fail()
217 tr = hwq->act_tr[i]; in ufshci_req_queue_fail()
218 req = tr->req; in ufshci_req_queue_fail()
220 if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED) { in ufshci_req_queue_fail()
221 mtx_unlock(&hwq->qlock); in ufshci_req_queue_fail()
225 mtx_lock(&hwq->qlock); in ufshci_req_queue_fail()
226 } else if (tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED) { in ufshci_req_queue_fail()
231 mtx_unlock(&hwq->qlock); in ufshci_req_queue_fail()
235 mtx_lock(&hwq->qlock); in ufshci_req_queue_fail()
239 mtx_unlock(&hwq->qlock); in ufshci_req_queue_fail()
245 struct ufshci_req_queue *req_queue = tr->req_queue; in ufshci_req_queue_complete_tracker()
246 struct ufshci_hw_queue *hwq = tr->hwq; in ufshci_req_queue_complete_tracker()
247 struct ufshci_request *req = tr->req; in ufshci_req_queue_complete_tracker()
252 mtx_assert(&hwq->qlock, MA_NOTOWNED); in ufshci_req_queue_complete_tracker()
256 cpl.size = tr->response_size; in ufshci_req_queue_complete_tracker()
257 if (req_queue->is_task_mgmt) { in ufshci_req_queue_complete_tracker()
259 (void *)hwq->utmrd[tr->slot_num].response_upiu, cpl.size); in ufshci_req_queue_complete_tracker()
261 ocs = hwq->utmrd[tr->slot_num].overall_command_status; in ufshci_req_queue_complete_tracker()
263 bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, in ufshci_req_queue_complete_tracker()
266 memcpy(&cpl.response_upiu, (void *)tr->ucd->response_upiu, in ufshci_req_queue_complete_tracker()
269 ocs = hwq->utrd[tr->slot_num].overall_command_status; in ufshci_req_queue_complete_tracker()
279 req->retries < req_queue->ctrlr->retry_count; in ufshci_req_queue_complete_tracker()
281 hwq->num_retries++; in ufshci_req_queue_complete_tracker()
282 if (error && req->retries >= req_queue->ctrlr->retry_count && retriable) in ufshci_req_queue_complete_tracker()
283 hwq->num_failures++; in ufshci_req_queue_complete_tracker()
285 KASSERT(tr->req, ("there is no request assigned to the tracker\n")); in ufshci_req_queue_complete_tracker()
287 req->request_upiu.header.task_tag, in ufshci_req_queue_complete_tracker()
291 if (req->payload_valid) { in ufshci_req_queue_complete_tracker()
292 bus_dmamap_sync(req_queue->dma_tag_payload, in ufshci_req_queue_complete_tracker()
293 tr->payload_dma_map, in ufshci_req_queue_complete_tracker()
297 if (req->cb_fn) in ufshci_req_queue_complete_tracker()
298 req->cb_fn(req->cb_arg, &cpl, error); in ufshci_req_queue_complete_tracker()
301 mtx_lock(&hwq->qlock); in ufshci_req_queue_complete_tracker()
304 req_queue->qops.clear_cpl_ntf(req_queue->ctrlr, tr); in ufshci_req_queue_complete_tracker()
307 req->retries++; in ufshci_req_queue_complete_tracker()
309 req->data_direction); in ufshci_req_queue_complete_tracker()
311 if (req->payload_valid) { in ufshci_req_queue_complete_tracker()
312 bus_dmamap_unload(req_queue->dma_tag_payload, in ufshci_req_queue_complete_tracker()
313 tr->payload_dma_map); in ufshci_req_queue_complete_tracker()
318 tr->req = NULL; in ufshci_req_queue_complete_tracker()
319 tr->slot_state = UFSHCI_SLOT_STATE_FREE; in ufshci_req_queue_complete_tracker()
321 TAILQ_REMOVE(&hwq->outstanding_tr, tr, tailq); in ufshci_req_queue_complete_tracker()
322 TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq); in ufshci_req_queue_complete_tracker()
325 mtx_unlock(&tr->hwq->qlock); in ufshci_req_queue_complete_tracker()
334 hwq = req_queue->qops.get_hw_queue(req_queue); in ufshci_req_queue_process_completions()
336 mtx_lock(&hwq->recovery_lock); in ufshci_req_queue_process_completions()
337 done = req_queue->qops.process_cpl(req_queue); in ufshci_req_queue_process_completions()
338 mtx_unlock(&hwq->recovery_lock); in ufshci_req_queue_process_completions()
356 ufshci_printf(tr->req_queue->ctrlr, in ufshci_payload_map()
361 prdt_entry = (struct ufshci_prdt_entry *)tr->ucd->prd_table; in ufshci_payload_map()
363 tr->prdt_entry_cnt = nseg; in ufshci_payload_map()
366 prdt_entry->data_base_address = htole64(seg[i].ds_addr) & in ufshci_payload_map()
368 prdt_entry->data_base_address_upper = htole64(seg[i].ds_addr) >> in ufshci_payload_map()
370 prdt_entry->data_byte_count = htole32(seg[i].ds_len - 1); in ufshci_payload_map()
375 bus_dmamap_sync(tr->req_queue->dma_tag_payload, tr->payload_dma_map, in ufshci_payload_map()
382 struct ufshci_request *req = tr->req; in ufshci_req_queue_prepare_prdt()
383 struct ufshci_utp_cmd_desc *cmd_desc = tr->ucd; in ufshci_req_queue_prepare_prdt()
386 tr->prdt_off = UFSHCI_UTP_XFER_REQ_SIZE + UFSHCI_UTP_XFER_RESP_SIZE; in ufshci_req_queue_prepare_prdt()
388 memset(cmd_desc->prd_table, 0, sizeof(cmd_desc->prd_table)); in ufshci_req_queue_prepare_prdt()
391 error = bus_dmamap_load_mem(tr->req_queue->dma_tag_payload, in ufshci_req_queue_prepare_prdt()
392 tr->payload_dma_map, &req->payload, ufshci_payload_map, tr, in ufshci_req_queue_prepare_prdt()
402 ufshci_printf(tr->req_queue->ctrlr, in ufshci_req_queue_prepare_prdt()
405 mtx_unlock(&tr->hwq->qlock); in ufshci_req_queue_prepare_prdt()
409 mtx_lock(&tr->hwq->qlock); in ufshci_req_queue_prepare_prdt()
418 desc->interrupt = true; in ufshci_req_queue_fill_utmr_descriptor()
420 desc->overall_command_status = UFSHCI_UTMR_OCS_INVALID; in ufshci_req_queue_fill_utmr_descriptor()
422 memcpy(desc->request_upiu, &req->request_upiu, req->request_size); in ufshci_req_queue_fill_utmr_descriptor()
436 * Set command type to UFS storage. in ufshci_req_queue_fill_utr_descriptor()
437 * The UFS 4.1 spec only defines 'UFS Storage' as a command type. in ufshci_req_queue_fill_utr_descriptor()
442 desc->command_type = command_type; in ufshci_req_queue_fill_utr_descriptor()
443 desc->data_direction = data_direction; in ufshci_req_queue_fill_utr_descriptor()
444 desc->interrupt = true; in ufshci_req_queue_fill_utr_descriptor()
446 desc->overall_command_status = UFSHCI_UTR_OCS_INVALID; in ufshci_req_queue_fill_utr_descriptor()
447 desc->utp_command_descriptor_base_address = (uint32_t)(paddr & in ufshci_req_queue_fill_utr_descriptor()
449 desc->utp_command_descriptor_base_address_upper = (uint32_t)(paddr >> in ufshci_req_queue_fill_utr_descriptor()
452 desc->response_upiu_offset = response_off / dword_size; in ufshci_req_queue_fill_utr_descriptor()
453 desc->response_upiu_length = response_len / dword_size; in ufshci_req_queue_fill_utr_descriptor()
454 desc->prdt_offset = prdt_off / dword_size; in ufshci_req_queue_fill_utr_descriptor()
455 desc->prdt_length = prdt_entry_cnt; in ufshci_req_queue_fill_utr_descriptor()
472 hwq->recovery_state = RECOVERY_WAITING; in ufshci_req_queue_timeout_recovery()
492 mtx_lock(&tr->hwq->qlock); in ufshci_abort_complete()
493 if (tr->slot_state != UFSHCI_SLOT_STATE_FREE) { in ufshci_abort_complete()
494 mtx_unlock(&tr->hwq->qlock); in ufshci_abort_complete()
501 ufshci_printf(tr->hwq->ctrlr, in ufshci_abort_complete()
506 if ((status->response_upiu.task_mgmt_response_upiu in ufshci_abort_complete()
509 (status->response_upiu.task_mgmt_response_upiu in ufshci_abort_complete()
512 ufshci_printf(tr->hwq->ctrlr, in ufshci_abort_complete()
518 /* Abort Task failed. Perform recovery steps 2-5 */ in ufshci_abort_complete()
519 ufshci_req_queue_timeout_recovery(tr->hwq->ctrlr, tr->hwq); in ufshci_abort_complete()
521 mtx_unlock(&tr->hwq->qlock); in ufshci_abort_complete()
529 struct ufshci_controller *ctrlr = hwq->ctrlr; in ufshci_req_queue_timeout()
535 mtx_assert(&hwq->recovery_lock, MA_OWNED); in ufshci_req_queue_timeout()
541 if (ctrlr->is_failed) { in ufshci_req_queue_timeout()
544 hwq->timer_armed = false; in ufshci_req_queue_timeout()
549 * Shutdown condition: We set hwq->timer_armed to false in in ufshci_req_queue_timeout()
555 if (!hwq->timer_armed) { in ufshci_req_queue_timeout()
561 switch (hwq->recovery_state) { in ufshci_req_queue_timeout()
569 mtx_lock(&hwq->qlock); in ufshci_req_queue_timeout()
571 TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) { in ufshci_req_queue_timeout()
577 if (now <= tr->deadline) in ufshci_req_queue_timeout()
581 mtx_unlock(&hwq->qlock); in ufshci_req_queue_timeout()
587 * deadline has passed. Poll the competions as a last-ditch in ufshci_req_queue_timeout()
590 hwq->req_queue->qops.process_cpl(hwq->req_queue); in ufshci_req_queue_timeout()
593 * Now that we've run the ISR, re-rheck to see if there's any in ufshci_req_queue_timeout()
596 mtx_lock(&hwq->qlock); in ufshci_req_queue_timeout()
598 TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) { in ufshci_req_queue_timeout()
607 if (now <= tr->deadline) in ufshci_req_queue_timeout()
624 if (ctrlr->enable_aborts && in ufshci_req_queue_timeout()
625 !hwq->req_queue->is_task_mgmt && in ufshci_req_queue_timeout()
626 tr->req->cb_fn != ufshci_abort_complete) { in ufshci_req_queue_timeout()
637 tr->req->request_upiu.header.task_tag); in ufshci_req_queue_timeout()
641 tr->req->request_upiu.header.lun, in ufshci_req_queue_timeout()
642 tr->req->request_upiu.header.task_tag, 0); in ufshci_req_queue_timeout()
644 /* Recovery Step 2-5 */ in ufshci_req_queue_timeout()
650 mtx_unlock(&hwq->qlock); in ufshci_req_queue_timeout()
661 if (!device_is_suspended(ctrlr->dev)) in ufshci_req_queue_timeout()
671 callout_schedule_sbt(&hwq->timer, SBT_1S / 2, SBT_1S / 2, 0); in ufshci_req_queue_timeout()
673 hwq->timer_armed = false; in ufshci_req_queue_timeout()
684 struct ufshci_controller *ctrlr = req_queue->ctrlr; in ufshci_req_queue_submit_tracker()
685 struct ufshci_request *req = tr->req; in ufshci_req_queue_submit_tracker()
689 uint8_t slot_num = tr->slot_num; in ufshci_req_queue_submit_tracker()
692 hwq = req_queue->qops.get_hw_queue(req_queue); in ufshci_req_queue_submit_tracker()
694 mtx_assert(&hwq->qlock, MA_OWNED); in ufshci_req_queue_submit_tracker()
696 if (req->cb_fn == ufshci_completion_poll_cb) in ufshci_req_queue_submit_tracker()
699 timeout = ctrlr->timeout_period; in ufshci_req_queue_submit_tracker()
700 tr->deadline = getsbinuptime() + timeout * SBT_1S; in ufshci_req_queue_submit_tracker()
701 if (!hwq->timer_armed) { in ufshci_req_queue_submit_tracker()
702 hwq->timer_armed = true; in ufshci_req_queue_submit_tracker()
707 callout_reset_sbt_on(&hwq->timer, SBT_1S / 2, SBT_1S / 2, in ufshci_req_queue_submit_tracker()
708 ufshci_req_queue_timeout, hwq, hwq->cpu, 0); in ufshci_req_queue_submit_tracker()
711 if (req_queue->is_task_mgmt) { in ufshci_req_queue_submit_tracker()
713 ufshci_req_queue_fill_utmr_descriptor(&tr->hwq->utmrd[slot_num], in ufshci_req_queue_submit_tracker()
716 request_len = req->request_size; in ufshci_req_queue_submit_tracker()
718 response_len = req->response_size; in ufshci_req_queue_submit_tracker()
721 memcpy(tr->ucd, &req->request_upiu, request_len); in ufshci_req_queue_submit_tracker()
722 memset((uint8_t *)tr->ucd + response_off, 0, response_len); in ufshci_req_queue_submit_tracker()
725 if (req->payload_valid) in ufshci_req_queue_submit_tracker()
729 ucd_paddr = tr->ucd_bus_addr; in ufshci_req_queue_submit_tracker()
730 ufshci_req_queue_fill_utr_descriptor(&tr->hwq->utrd[slot_num], in ufshci_req_queue_submit_tracker()
732 tr->prdt_off, tr->prdt_entry_cnt); in ufshci_req_queue_submit_tracker()
734 bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, in ufshci_req_queue_submit_tracker()
738 bus_dmamap_sync(tr->hwq->dma_tag_queue, tr->hwq->queuemem_map, in ufshci_req_queue_submit_tracker()
741 tr->slot_state = UFSHCI_SLOT_STATE_SCHEDULED; in ufshci_req_queue_submit_tracker()
744 req_queue->qops.ring_doorbell(ctrlr, tr); in ufshci_req_queue_submit_tracker()
754 mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED); in _ufshci_req_queue_submit_request()
756 error = req_queue->qops.reserve_slot(req_queue, &tr); in _ufshci_req_queue_submit_request()
758 ufshci_printf(req_queue->ctrlr, "Failed to get tracker"); in _ufshci_req_queue_submit_request()
763 if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED || in _ufshci_req_queue_submit_request()
764 tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED) in _ufshci_req_queue_submit_request()
768 req->request_upiu.header.task_tag = tr->slot_num; in _ufshci_req_queue_submit_request()
770 tr->slot_state = UFSHCI_SLOT_STATE_RESERVED; in _ufshci_req_queue_submit_request()
771 tr->response_size = req->response_size; in _ufshci_req_queue_submit_request()
772 tr->deadline = SBT_MAX; in _ufshci_req_queue_submit_request()
773 tr->req = req; in _ufshci_req_queue_submit_request()
775 TAILQ_REMOVE(&tr->hwq->free_tr, tr, tailq); in _ufshci_req_queue_submit_request()
776 TAILQ_INSERT_TAIL(&tr->hwq->outstanding_tr, tr, tailq); in _ufshci_req_queue_submit_request()
778 ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction); in _ufshci_req_queue_submit_request()
792 hwq = req_queue->qops.get_hw_queue(req_queue); in ufshci_req_queue_submit_request()
795 mtx_lock(&hwq->qlock); in ufshci_req_queue_submit_request()
797 mtx_unlock(&hwq->qlock); in ufshci_req_queue_submit_request()