dev/ufshci/ufshci_req_queue.c

/*-
 * Copyright (c) 2025, Samsung Electronics Co., Ltd.
 * Written by Jaeyoon Choi
 *
 * SPDX-License-Identifier: BSD-2-Clause
 */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/domainset.h>
#include <sys/module.h>

#include <cam/scsi/scsi_all.h>

#include "sys/kassert.h"
#include "ufshci_private.h"

static void ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,
    struct ufshci_tracker *tr, enum ufshci_data_direction data_direction);

static const struct ufshci_qops sdb_utmr_qops = {
	.construct = ufshci_req_sdb_construct,
	.destroy = ufshci_req_sdb_destroy,
	.get_hw_queue = ufshci_req_sdb_get_hw_queue,
	.enable = ufshci_req_sdb_enable,
	.disable = ufshci_req_sdb_disable,
	.reserve_slot = ufshci_req_sdb_reserve_slot,
	.reserve_admin_slot = ufshci_req_sdb_reserve_slot,
	.ring_doorbell = ufshci_req_sdb_utmr_ring_doorbell,
	.is_doorbell_cleared = ufshci_req_sdb_utmr_is_doorbell_cleared,
	.clear_cpl_ntf = ufshci_req_sdb_utmr_clear_cpl_ntf,
	.process_cpl = ufshci_req_sdb_process_cpl,
	.get_inflight_io = ufshci_req_sdb_get_inflight_io,
};

static const struct ufshci_qops sdb_utr_qops = {
	.construct = ufshci_req_sdb_construct,
	.destroy = ufshci_req_sdb_destroy,
	.get_hw_queue = ufshci_req_sdb_get_hw_queue,
	.enable = ufshci_req_sdb_enable,
	.disable = ufshci_req_sdb_disable,
	.reserve_slot = ufshci_req_sdb_reserve_slot,
	.reserve_admin_slot = ufshci_req_sdb_reserve_slot,
	.ring_doorbell = ufshci_req_sdb_utr_ring_doorbell,
	.is_doorbell_cleared = ufshci_req_sdb_utr_is_doorbell_cleared,
	.clear_cpl_ntf = ufshci_req_sdb_utr_clear_cpl_ntf,
	.process_cpl = ufshci_req_sdb_process_cpl,
	.get_inflight_io = ufshci_req_sdb_get_inflight_io,
};

int
ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr)
{
	struct ufshci_req_queue *req_queue;
	int error;

	/*
	 * UTP Task Management Request only supports Legacy Single Doorbell
	 * Queue.
	 */
	req_queue = &ctrlr->task_mgmt_req_queue;
	req_queue->queue_mode = UFSHCI_Q_MODE_SDB;
	req_queue->qops = sdb_utmr_qops;

	error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES,
	    /*is_task_mgmt*/ true);

	return (error);
}

void
ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr)
{
	/* Attach may fail before the queue ops are set up. */
	if (ctrlr->task_mgmt_req_queue.qops.destroy == NULL)
		return;

	ctrlr->task_mgmt_req_queue.qops.destroy(ctrlr,
	    &ctrlr->task_mgmt_req_queue);
}

void
ufshci_utmr_req_queue_disable(struct ufshci_controller *ctrlr)
{
	ctrlr->task_mgmt_req_queue.qops.disable(ctrlr,
	    &ctrlr->task_mgmt_req_queue);
}

int
ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr)
{
	return (ctrlr->task_mgmt_req_queue.qops.enable(ctrlr,
	    &ctrlr->task_mgmt_req_queue));
}

int
ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr)
{
	struct ufshci_req_queue *req_queue;
	int error;

	/*
	 * Currently, it does not support MCQ mode, so it should be set to SDB
	 * mode by default.
	 * TODO: Determine queue mode by checking Capability Registers
	 */
	req_queue = &ctrlr->transfer_req_queue;
	req_queue->queue_mode = UFSHCI_Q_MODE_SDB;
	req_queue->qops = sdb_utr_qops;

	error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES,
	    /*is_task_mgmt*/ false);

	return (error);
}

void
ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr)
{
	/* Attach may fail before the queue ops are set up. */
	if (ctrlr->transfer_req_queue.qops.destroy == NULL)
		return;

	ctrlr->transfer_req_queue.qops.destroy(ctrlr,
	    &ctrlr->transfer_req_queue);
}

void
ufshci_utr_req_queue_disable(struct ufshci_controller *ctrlr)
{
	ctrlr->transfer_req_queue.qops.disable(ctrlr,
	    &ctrlr->transfer_req_queue);
}

int
ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr)
{
	return (ctrlr->transfer_req_queue.qops.enable(ctrlr,
	    &ctrlr->transfer_req_queue));
}

static bool
ufshci_req_queue_response_is_error(struct ufshci_req_queue *req_queue,
    uint8_t ocs, union ufshci_reponse_upiu *response)
{
	bool is_error = false;

	/* Check request descriptor */
	if (ocs != UFSHCI_DESC_SUCCESS) {
		ufshci_printf(req_queue->ctrlr, "Invalid OCS = 0x%x\n", ocs);
		is_error = true;
	}

	/* Check response UPIU header */
	if (response->header.response != UFSHCI_RESPONSE_CODE_TARGET_SUCCESS) {
		ufshci_printf(req_queue->ctrlr,
		    "Function(0x%x) Invalid response code = 0x%x\n",
		    response->header.ext_iid_or_function,
		    response->header.response);
		is_error = true;
	}

	return (is_error);
}

static void
ufshci_req_queue_manual_complete_tracker(struct ufshci_tracker *tr, uint8_t ocs,
    uint8_t rc)
{
	struct ufshci_utp_xfer_req_desc *desc;
	struct ufshci_upiu_header *resp_header;

	mtx_assert(&tr->hwq->qlock, MA_NOTOWNED);

	resp_header = (struct ufshci_upiu_header *)tr->ucd->response_upiu;
	resp_header->response = rc;

	desc = &tr->hwq->utrd[tr->slot_num];
	desc->overall_command_status = ocs;

	ufshci_req_queue_complete_tracker(tr);
}

static void
ufshci_req_queue_manual_complete_request(struct ufshci_req_queue *req_queue,
    struct ufshci_request *req, uint8_t ocs, uint8_t rc)
{
	struct ufshci_completion cpl;
	bool error;

	memset(&cpl, 0, sizeof(cpl));
	cpl.response_upiu.header.response = rc;
	error = ufshci_req_queue_response_is_error(req_queue, ocs,
	    &cpl.response_upiu);

	if (error) {
		ufshci_printf(req_queue->ctrlr,
		    "Manual complete request error:0x%x", error);
	}

	if (req->cb_fn)
		req->cb_fn(req->cb_arg, &cpl, error);

	ufshci_free_request(req);
}

void
ufshci_req_queue_fail(struct ufshci_controller *ctrlr,
    struct ufshci_req_queue *req_queue)
{
	struct ufshci_hw_queue *hwq = req_queue->qops.get_hw_queue(req_queue);
	struct ufshci_tracker *tr;
	int i;

	if (!mtx_initialized(&hwq->qlock))
		return;

	mtx_lock(&hwq->qlock);

	for (i = 0; i < req_queue->num_trackers; i++) {
		tr = hwq->act_tr[i];

		if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED) {
			mtx_unlock(&hwq->qlock);
			ufshci_req_queue_manual_complete_request(req_queue,
			    tr->req, UFSHCI_DESC_ABORTED,
			    UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);
			mtx_lock(&hwq->qlock);
		} else if (tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED) {
			/*
			 * Do not remove the tracker. The abort_tracker path
			 * will do that for us.
			 */
			mtx_unlock(&hwq->qlock);
			ufshci_req_queue_manual_complete_tracker(tr,
			    UFSHCI_DESC_ABORTED,
			    UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);
			mtx_lock(&hwq->qlock);
		}
	}

	mtx_unlock(&hwq->qlock);
}

void
ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr)
{
	struct ufshci_req_queue *req_queue = tr->req_queue;
	struct ufshci_hw_queue *hwq = tr->hwq;
	struct ufshci_request *req = tr->req;
	struct ufshci_completion cpl;
	uint8_t ocs;
	bool retry, error, retriable;

	mtx_assert(&hwq->qlock, MA_NOTOWNED);

	/* Copy the response from the Request Descriptor or UTP Command
	 * Descriptor. */
	cpl.size = tr->response_size;
	if (req_queue->is_task_mgmt) {
		memcpy(&cpl.response_upiu,
		    (void *)hwq->utmrd[tr->slot_num].response_upiu, cpl.size);

		ocs = hwq->utmrd[tr->slot_num].overall_command_status;
	} else {
		bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,
		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

		memcpy(&cpl.response_upiu, (void *)tr->ucd->response_upiu,
		    cpl.size);

		ocs = hwq->utrd[tr->slot_num].overall_command_status;
	}

	error = ufshci_req_queue_response_is_error(req_queue, ocs,
	    &cpl.response_upiu);

	/* Retry for admin commands */
	retriable = req->is_admin;
	retry = error && retriable &&
	    req->retries < req_queue->ctrlr->retry_count;
	if (retry)
		hwq->num_retries++;
	if (error && req->retries >= req_queue->ctrlr->retry_count && retriable)
		hwq->num_failures++;

	KASSERT(tr->req, ("there is no request assigned to the tracker\n"));
	KASSERT(cpl.response_upiu.header.task_tag ==
		req->request_upiu.header.task_tag,
	    ("response task_tag does not match request task_tag\n"));

	if (!retry) {
		if (req->payload_valid) {
			bus_dmamap_sync(req_queue->dma_tag_payload,
			    tr->payload_dma_map,
			    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
		}
		/* Copy response from the command descriptor */
		if (req->cb_fn)
			req->cb_fn(req->cb_arg, &cpl, error);
	}

	mtx_lock(&hwq->qlock);

	/* Clear the UTRL Completion Notification register */
	req_queue->qops.clear_cpl_ntf(req_queue->ctrlr, tr);

	if (retry) {
		req->retries++;
		ufshci_req_queue_submit_tracker(req_queue, tr,
		    req->data_direction);
	} else {
		if (req->payload_valid) {
			bus_dmamap_unload(req_queue->dma_tag_payload,
			    tr->payload_dma_map);
		}

		/* Clear tracker */
		ufshci_free_request(req);
		tr->req = NULL;
		tr->slot_state = UFSHCI_SLOT_STATE_FREE;

		TAILQ_REMOVE(&hwq->outstanding_tr, tr, tailq);
		TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq);
	}

	mtx_unlock(&tr->hwq->qlock);
}

bool
ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue)
{
	struct ufshci_hw_queue *hwq;
	bool done;

	hwq = req_queue->qops.get_hw_queue(req_queue);

	mtx_lock(&hwq->recovery_lock);
	done = req_queue->qops.process_cpl(req_queue);
	mtx_unlock(&hwq->recovery_lock);

	return (done);
}

static void
ufshci_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
{
	struct ufshci_tracker *tr = arg;
	struct ufshci_prdt_entry *prdt_entry;
	int i;

	/*
	 * If the mapping operation failed, return immediately. The caller
	 * is responsible for detecting the error status and failing the
	 * tracker manually.
	 */
	if (error != 0) {
		ufshci_printf(tr->req_queue->ctrlr,
		    "Failed to map payload %d\n", error);
		return;
	}

	prdt_entry = (struct ufshci_prdt_entry *)tr->ucd->prd_table;

	tr->prdt_entry_cnt = nseg;

	for (i = 0; i < nseg; i++) {
		prdt_entry->data_base_address = htole64(seg[i].ds_addr) &
		    0xffffffff;
		prdt_entry->data_base_address_upper = htole64(seg[i].ds_addr) >>
		    32;
		prdt_entry->data_byte_count = htole32(seg[i].ds_len - 1);

		++prdt_entry;
	}

	bus_dmamap_sync(tr->req_queue->dma_tag_payload, tr->payload_dma_map,
	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}

static int
ufshci_req_queue_prepare_prdt(struct ufshci_tracker *tr)
{
	struct ufshci_request *req = tr->req;
	struct ufshci_utp_cmd_desc *cmd_desc = tr->ucd;
	int error;

	tr->prdt_off = UFSHCI_UTP_XFER_REQ_SIZE + UFSHCI_UTP_XFER_RESP_SIZE;

	memset(cmd_desc->prd_table, 0, sizeof(cmd_desc->prd_table));

	/* Filling PRDT enrties with payload */
	error = bus_dmamap_load_mem(tr->req_queue->dma_tag_payload,
	    tr->payload_dma_map, &req->payload, ufshci_payload_map, tr,
	    BUS_DMA_NOWAIT);
	if (error != 0) {
		/*
		 * The dmamap operation failed, so we manually fail the
		 *  tracker here with UFSHCI_DESC_INVALID_PRDT_ATTRIBUTES.
		 *
		 * ufshci_req_queue_manual_complete_tracker must not be called
		 *  with the req_queue lock held.
		 */
		ufshci_printf(tr->req_queue->ctrlr,
		    "bus_dmamap_load_mem returned with error:0x%x!\n", error);

		mtx_unlock(&tr->hwq->qlock);
		ufshci_req_queue_manual_complete_tracker(tr,
		    UFSHCI_DESC_INVALID_PRDT_ATTRIBUTES,
		    UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);
		mtx_lock(&tr->hwq->qlock);
	}

	return (error);
}

static void
ufshci_req_queue_fill_utmr_descriptor(
    struct ufshci_utp_task_mgmt_req_desc *desc, struct ufshci_request *req)
{
	memset(desc, 0, sizeof(struct ufshci_utp_task_mgmt_req_desc));
	desc->interrupt = true;
	/* Set the initial value to Invalid. */
	desc->overall_command_status = UFSHCI_UTMR_OCS_INVALID;

	memcpy(desc->request_upiu, &req->request_upiu, req->request_size);
}

static void
ufshci_req_queue_fill_utr_descriptor(struct ufshci_utp_xfer_req_desc *desc,
    uint8_t data_direction, const uint64_t paddr, const uint16_t response_off,
    const uint16_t response_len, const uint16_t prdt_off,
    const uint16_t prdt_entry_cnt)
{
	uint8_t command_type;
	/* Value to convert bytes to dwords */
	const uint16_t dword_size = 4;

	/*
	 * Set command type to UFS storage.
	 * The UFS 4.1 spec only defines 'UFS Storage' as a command type.
	 */
	command_type = UFSHCI_COMMAND_TYPE_UFS_STORAGE;

	memset(desc, 0, sizeof(struct ufshci_utp_xfer_req_desc));
	desc->command_type = command_type;
	desc->data_direction = data_direction;
	desc->interrupt = true;
	/* Set the initial value to Invalid. */
	desc->overall_command_status = UFSHCI_UTR_OCS_INVALID;
	desc->utp_command_descriptor_base_address = (uint32_t)(paddr &
	    0xffffffff);
	desc->utp_command_descriptor_base_address_upper = (uint32_t)(paddr >>
	    32);

	desc->response_upiu_offset = response_off / dword_size;
	desc->response_upiu_length = response_len / dword_size;
	desc->prdt_offset = prdt_off / dword_size;
	desc->prdt_length = prdt_entry_cnt;
}

static void
ufshci_req_queue_timeout_recovery(struct ufshci_controller *ctrlr,
    struct ufshci_hw_queue *hwq)
{
	/* TODO: Step 2. Logical unit reset */
	/* TODO: Step 3. Target device reset */
	/* TODO: Step 4. Bus reset */

	/*
	 * Step 5. All previous commands were timeout.
	 * Recovery failed, reset the host controller.
	 */
	ufshci_printf(ctrlr,
	    "Recovery step 5: Resetting controller due to a timeout.\n");
	hwq->recovery_state = RECOVERY_WAITING;

	ufshci_ctrlr_reset(ctrlr);
}

static void
ufshci_abort_complete(void *arg, const struct ufshci_completion *status,
    bool error)
{
	struct ufshci_tracker *tr = arg;
	uint32_t output_param1;

	/*
	 * We still need to check the active tracker array, to cover race where
	 * I/O timed out at same time controller was completing the I/O. An
	 * abort request always is on the Task Management Request queue, but
	 * affects either an Task Management Request or an I/O (UTRL) queue, so
	 * take the appropriate queue lock for the original command's queue,
	 * since we'll need it to avoid races with the completion code and to
	 * complete the command manually.
	 */
	mtx_lock(&tr->hwq->qlock);
	if (tr->slot_state != UFSHCI_SLOT_STATE_FREE) {
		mtx_unlock(&tr->hwq->qlock);
		/*
		 * An I/O has timed out, and the controller was unable to abort
		 * it for some reason.  And we've not processed a completion for
		 * it yet. Construct a fake completion status, and then complete
		 * the I/O's tracker manually.
		 */
		ufshci_printf(tr->hwq->ctrlr,
		    "abort task request failed, aborting task manually\n");
		ufshci_req_queue_manual_complete_tracker(tr,
		    UFSHCI_DESC_ABORTED, UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);

		output_param1 = be32toh(
		    status->response_upiu.task_mgmt_response_upiu.output_param1);
		if (output_param1 ==
			UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_COMPLETE ||
		    output_param1 ==
			UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_SUCCEEDED) {
			ufshci_printf(tr->hwq->ctrlr,
			    "Warning: the abort task request completed \
			    successfully, but the original task is still incomplete.");
			return;
		}

		/* Abort Task failed. Perform recovery steps 2-5 */
		ufshci_req_queue_timeout_recovery(tr->hwq->ctrlr, tr->hwq);
	} else {
		mtx_unlock(&tr->hwq->qlock);
	}
}

static void
ufshci_req_queue_timeout(void *arg)
{
	struct ufshci_hw_queue *hwq = arg;
	struct ufshci_controller *ctrlr = hwq->ctrlr;
	struct ufshci_tracker *tr;
	sbintime_t now;
	bool idle = true;
	bool fast;

	mtx_assert(&hwq->recovery_lock, MA_OWNED);

	/*
	 * If the controller is failed, then stop polling. This ensures that any
	 * failure processing that races with the hwq timeout will fail safely.
	 */
	if (ctrlr->is_failed) {
		ufshci_printf(ctrlr,
		    "Failed controller, stopping watchdog timeout.\n");
		hwq->timer_armed = false;
		return;
	}

	/*
	 * Shutdown condition: We set hwq->timer_armed to false in
	 * ufshci_req_sdb_destroy before calling callout_drain. When we call
	 * that, this routine might get called one last time. Exit w/o setting a
	 * timeout. None of the watchdog stuff needs to be done since we're
	 * destroying the hwq.
	 */
	if (!hwq->timer_armed) {
		ufshci_printf(ctrlr,
		    "Timeout fired during ufshci_utr_req_queue_destroy\n");
		return;
	}

	switch (hwq->recovery_state) {
	case RECOVERY_NONE:
		/*
		 * See if there's any recovery needed. First, do a fast check to
		 * see if anything could have timed out. If not, then skip
		 * everything else.
		 */
		fast = false;
		mtx_lock(&hwq->qlock);
		now = getsbinuptime();
		TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) {
			/*
			 * If the first real transaction is not in timeout, then
			 * we're done. Otherwise, we try recovery.
			 */
			idle = false;
			if (now <= tr->deadline)
				fast = true;
			break;
		}
		mtx_unlock(&hwq->qlock);
		if (idle || fast)
			break;

		/*
		 * There's a stale transaction at the start of the queue whose
		 * deadline has passed. Poll the competions as a last-ditch
		 * effort in case an interrupt has been missed.
		 */
		hwq->req_queue->qops.process_cpl(hwq->req_queue);

		/*
		 * Now that we've run the ISR, re-rheck to see if there's any
		 * timed out commands and abort them or reset the card if so.
		 */
		mtx_lock(&hwq->qlock);
		idle = true;
		TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) {
			/*
			 * If we know this tracker hasn't timed out, we also
			 * know all subsequent ones haven't timed out. The tr
			 * queue is in submission order and all normal commands
			 * in a queue have the same timeout (or the timeout was
			 * changed by the user, but we eventually timeout then).
			 */
			idle = false;
			if (now <= tr->deadline)
				break;

			/*
			 * Timeout recovery is performed in five steps. If
			 * recovery fails at any step, the process continues to
			 * the next one:
			 * next steps:
			 * Step 1. Abort task
			 * Step 2. Logical unit reset 	(TODO)
			 * Step 3. Target device reset 	(TODO)
			 * Step 4. Bus reset 		(TODO)
			 * Step 5. Host controller reset
			 *
			 * If the timeout occurred in the Task Management
			 * Request queue, ignore Step 1.
			 */
			if (ctrlr->enable_aborts &&
			    !hwq->req_queue->is_task_mgmt &&
			    tr->req->cb_fn != ufshci_abort_complete) {
				/*
				 * Step 1. Timeout expired, abort the task.
				 *
				 * This isn't an abort command, ask for a
				 * hardware abort. This goes to the Task
				 * Management Request queue which will reset the
				 * task if it times out.
				 */
				ufshci_printf(ctrlr,
				    "Recovery step 1: Timeout occurred. aborting the task(%d).\n",
				    tr->req->request_upiu.header.task_tag);
				if (ufshci_ctrlr_cmd_send_task_mgmt_request(ctrlr,
					ufshci_abort_complete, tr,
					UFSHCI_TASK_MGMT_FUNCTION_ABORT_TASK,
					tr->req->request_upiu.header.lun,
					tr->req->request_upiu.header.task_tag,
					0) != 0) {
					ufshci_req_queue_timeout_recovery(ctrlr,
					    hwq);
					idle = false;
					break;
				}
			} else {
				/* Recovery Step 2-5 */
				ufshci_req_queue_timeout_recovery(ctrlr, hwq);
				idle = false;
				break;
			}
		}
		mtx_unlock(&hwq->qlock);
		break;

	case RECOVERY_WAITING:
		/*
		 * These messages aren't interesting while we're suspended. We
		 * put the queues into waiting state while suspending.
		 * Suspending takes a while, so we'll see these during that time
		 * and they aren't diagnostic. At other times, they indicate a
		 * problem that's worth complaining about.
		 */
		if (!device_is_suspended(ctrlr->dev))
			ufshci_printf(ctrlr, "Waiting for reset to complete\n");
		idle = false; /* We want to keep polling */
		break;
	}

	/*
	 * Rearm the timeout.
	 */
	if (!idle) {
		callout_schedule_sbt(&hwq->timer, SBT_1S / 2, SBT_1S / 2, 0);
	} else {
		hwq->timer_armed = false;
	}
}

/*
 * Submit the tracker to the hardware.
 */
static void
ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,
    struct ufshci_tracker *tr, enum ufshci_data_direction data_direction)
{
	struct ufshci_controller *ctrlr = req_queue->ctrlr;
	struct ufshci_request *req = tr->req;
	struct ufshci_hw_queue *hwq;
	uint64_t ucd_paddr;
	uint16_t request_len, response_off, response_len;
	uint8_t slot_num = tr->slot_num;
	int timeout;

	hwq = req_queue->qops.get_hw_queue(req_queue);

	mtx_assert(&hwq->qlock, MA_OWNED);

	if (req->cb_fn == ufshci_completion_poll_cb)
		timeout = 1;
	else
		timeout = ctrlr->timeout_period;
	tr->deadline = getsbinuptime() + timeout * SBT_1S;
	if (!hwq->timer_armed) {
		hwq->timer_armed = true;
		/*
		 * It wakes up once every 0.5 seconds to check if the deadline
		 * has passed.
		 */
		callout_reset_sbt_on(&hwq->timer, SBT_1S / 2, SBT_1S / 2,
		    ufshci_req_queue_timeout, hwq, hwq->cpu, 0);
	}

	if (req_queue->is_task_mgmt) {
		/* Prepare UTP Task Management Request Descriptor. */
		ufshci_req_queue_fill_utmr_descriptor(&tr->hwq->utmrd[slot_num],
		    req);
	} else {
		request_len = req->request_size;
		response_off = UFSHCI_UTP_XFER_REQ_SIZE;
		response_len = req->response_size;

		/* Prepare UTP Command Descriptor */
		memcpy(tr->ucd, &req->request_upiu, request_len);
		memset((uint8_t *)tr->ucd + response_off, 0, response_len);

		/*
		 * Prepare PRDT. If the payload could not be mapped, the
		 * tracker has already been completed and released by the
		 * manual completion path, so the descriptor must not be
		 * built and the doorbell must not be rung.
		 */
		if (req->payload_valid &&
		    ufshci_req_queue_prepare_prdt(tr) != 0)
			return;

		/* Prepare UTP Transfer Request Descriptor. */
		ucd_paddr = tr->ucd_bus_addr;
		ufshci_req_queue_fill_utr_descriptor(&tr->hwq->utrd[slot_num],
		    data_direction, ucd_paddr, response_off, response_len,
		    tr->prdt_off, tr->prdt_entry_cnt);

		bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,
		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
	}

	bus_dmamap_sync(tr->hwq->dma_tag_queue, tr->hwq->queuemem_map,
	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

	tr->slot_state = UFSHCI_SLOT_STATE_SCHEDULED;

	/* Ring the doorbell */
	req_queue->qops.ring_doorbell(ctrlr, tr);
}

static int
_ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
    struct ufshci_request *req)
{
	struct ufshci_tracker *tr = NULL;
	int error;

	mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED);

	error = req_queue->qops.reserve_slot(req_queue, &tr);
	if (error != 0) {
		ufshci_printf(req_queue->ctrlr, "Failed to get tracker");
		return (error);
	}
	KASSERT(tr, ("There is no tracker allocated."));

	if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED ||
	    tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED)
		return (EBUSY);

	/* Set the task_tag value to slot_num for traceability. */
	req->request_upiu.header.task_tag = tr->slot_num;

	tr->slot_state = UFSHCI_SLOT_STATE_RESERVED;
	tr->response_size = req->response_size;
	tr->deadline = SBT_MAX;
	tr->req = req;

	TAILQ_REMOVE(&tr->hwq->free_tr, tr, tailq);
	TAILQ_INSERT_TAIL(&tr->hwq->outstanding_tr, tr, tailq);

	ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction);

	return (0);
}

int
ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
    struct ufshci_request *req)
{
	struct ufshci_hw_queue *hwq;
	uint32_t error;

	/* TODO: MCQs should use a separate Admin queue. */

	hwq = req_queue->qops.get_hw_queue(req_queue);
	KASSERT(hwq, ("There is no HW queue allocated."));

	mtx_lock(&hwq->qlock);
	error = _ufshci_req_queue_submit_request(req_queue, req);
	mtx_unlock(&hwq->qlock);

	return (error);
}