1 /*-
2 * Copyright (c) 2025, Samsung Electronics Co., Ltd.
3 * Written by Jaeyoon Choi
4 *
5 * SPDX-License-Identifier: BSD-2-Clause
6 */
7
8 #include <sys/param.h>
9 #include <sys/bus.h>
10 #include <sys/conf.h>
11 #include <sys/domainset.h>
12 #include <sys/module.h>
13
14 #include <cam/scsi/scsi_all.h>
15
16 #include "sys/kassert.h"
17 #include "ufshci_private.h"
18
19 static void ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,
20 struct ufshci_tracker *tr, enum ufshci_data_direction data_direction);
21
22 static const struct ufshci_qops sdb_utmr_qops = {
23 .construct = ufshci_req_sdb_construct,
24 .destroy = ufshci_req_sdb_destroy,
25 .get_hw_queue = ufshci_req_sdb_get_hw_queue,
26 .enable = ufshci_req_sdb_enable,
27 .disable = ufshci_req_sdb_disable,
28 .reserve_slot = ufshci_req_sdb_reserve_slot,
29 .reserve_admin_slot = ufshci_req_sdb_reserve_slot,
30 .ring_doorbell = ufshci_req_sdb_utmr_ring_doorbell,
31 .is_doorbell_cleared = ufshci_req_sdb_utmr_is_doorbell_cleared,
32 .clear_cpl_ntf = ufshci_req_sdb_utmr_clear_cpl_ntf,
33 .process_cpl = ufshci_req_sdb_process_cpl,
34 .get_inflight_io = ufshci_req_sdb_get_inflight_io,
35 };
36
37 static const struct ufshci_qops sdb_utr_qops = {
38 .construct = ufshci_req_sdb_construct,
39 .destroy = ufshci_req_sdb_destroy,
40 .get_hw_queue = ufshci_req_sdb_get_hw_queue,
41 .enable = ufshci_req_sdb_enable,
42 .disable = ufshci_req_sdb_disable,
43 .reserve_slot = ufshci_req_sdb_reserve_slot,
44 .reserve_admin_slot = ufshci_req_sdb_reserve_slot,
45 .ring_doorbell = ufshci_req_sdb_utr_ring_doorbell,
46 .is_doorbell_cleared = ufshci_req_sdb_utr_is_doorbell_cleared,
47 .clear_cpl_ntf = ufshci_req_sdb_utr_clear_cpl_ntf,
48 .process_cpl = ufshci_req_sdb_process_cpl,
49 .get_inflight_io = ufshci_req_sdb_get_inflight_io,
50 };
51
52 int
ufshci_utmr_req_queue_construct(struct ufshci_controller * ctrlr)53 ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr)
54 {
55 struct ufshci_req_queue *req_queue;
56 int error;
57
58 /*
59 * UTP Task Management Request only supports Legacy Single Doorbell
60 * Queue.
61 */
62 req_queue = &ctrlr->task_mgmt_req_queue;
63 req_queue->queue_mode = UFSHCI_Q_MODE_SDB;
64 req_queue->qops = sdb_utmr_qops;
65
66 error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES,
67 /*is_task_mgmt*/ true);
68
69 return (error);
70 }
71
72 void
ufshci_utmr_req_queue_destroy(struct ufshci_controller * ctrlr)73 ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr)
74 {
75 /* Attach may fail before the queue ops are set up. */
76 if (ctrlr->task_mgmt_req_queue.qops.destroy == NULL)
77 return;
78
79 ctrlr->task_mgmt_req_queue.qops.destroy(ctrlr,
80 &ctrlr->task_mgmt_req_queue);
81 }
82
83 void
ufshci_utmr_req_queue_disable(struct ufshci_controller * ctrlr)84 ufshci_utmr_req_queue_disable(struct ufshci_controller *ctrlr)
85 {
86 ctrlr->task_mgmt_req_queue.qops.disable(ctrlr,
87 &ctrlr->task_mgmt_req_queue);
88 }
89
90 int
ufshci_utmr_req_queue_enable(struct ufshci_controller * ctrlr)91 ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr)
92 {
93 return (ctrlr->task_mgmt_req_queue.qops.enable(ctrlr,
94 &ctrlr->task_mgmt_req_queue));
95 }
96
97 int
ufshci_utr_req_queue_construct(struct ufshci_controller * ctrlr)98 ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr)
99 {
100 struct ufshci_req_queue *req_queue;
101 int error;
102
103 /*
104 * Currently, it does not support MCQ mode, so it should be set to SDB
105 * mode by default.
106 * TODO: Determine queue mode by checking Capability Registers
107 */
108 req_queue = &ctrlr->transfer_req_queue;
109 req_queue->queue_mode = UFSHCI_Q_MODE_SDB;
110 req_queue->qops = sdb_utr_qops;
111
112 error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES,
113 /*is_task_mgmt*/ false);
114
115 return (error);
116 }
117
118 void
ufshci_utr_req_queue_destroy(struct ufshci_controller * ctrlr)119 ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr)
120 {
121 /* Attach may fail before the queue ops are set up. */
122 if (ctrlr->transfer_req_queue.qops.destroy == NULL)
123 return;
124
125 ctrlr->transfer_req_queue.qops.destroy(ctrlr,
126 &ctrlr->transfer_req_queue);
127 }
128
129 void
ufshci_utr_req_queue_disable(struct ufshci_controller * ctrlr)130 ufshci_utr_req_queue_disable(struct ufshci_controller *ctrlr)
131 {
132 ctrlr->transfer_req_queue.qops.disable(ctrlr,
133 &ctrlr->transfer_req_queue);
134 }
135
136 int
ufshci_utr_req_queue_enable(struct ufshci_controller * ctrlr)137 ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr)
138 {
139 return (ctrlr->transfer_req_queue.qops.enable(ctrlr,
140 &ctrlr->transfer_req_queue));
141 }
142
143 static bool
ufshci_req_queue_response_is_error(struct ufshci_req_queue * req_queue,uint8_t ocs,union ufshci_reponse_upiu * response)144 ufshci_req_queue_response_is_error(struct ufshci_req_queue *req_queue,
145 uint8_t ocs, union ufshci_reponse_upiu *response)
146 {
147 bool is_error = false;
148
149 /* Check request descriptor */
150 if (ocs != UFSHCI_DESC_SUCCESS) {
151 ufshci_printf(req_queue->ctrlr, "Invalid OCS = 0x%x\n", ocs);
152 is_error = true;
153 }
154
155 /* Check response UPIU header */
156 if (response->header.response != UFSHCI_RESPONSE_CODE_TARGET_SUCCESS) {
157 ufshci_printf(req_queue->ctrlr,
158 "Function(0x%x) Invalid response code = 0x%x\n",
159 response->header.ext_iid_or_function,
160 response->header.response);
161 is_error = true;
162 }
163
164 return (is_error);
165 }
166
167 static void
ufshci_req_queue_manual_complete_tracker(struct ufshci_tracker * tr,uint8_t ocs,uint8_t rc)168 ufshci_req_queue_manual_complete_tracker(struct ufshci_tracker *tr, uint8_t ocs,
169 uint8_t rc)
170 {
171 struct ufshci_req_queue *req_queue = tr->req_queue;
172 struct ufshci_hw_queue *hwq = tr->hwq;
173 struct ufshci_upiu_header *resp_header;
174
175 mtx_assert(&hwq->qlock, MA_NOTOWNED);
176
177 /*
178 * Write the fake response where the completion path reads it.
179 */
180 if (req_queue->is_task_mgmt) {
181 resp_header = (struct ufshci_upiu_header *)
182 hwq->utmrd[tr->slot_num].response_upiu;
183 hwq->utmrd[tr->slot_num].overall_command_status = ocs;
184 } else {
185 resp_header = (struct ufshci_upiu_header *)
186 tr->ucd->response_upiu;
187 hwq->utrd[tr->slot_num].overall_command_status = ocs;
188 }
189 resp_header->response = rc;
190 /*
191 * The hardware never wrote a response. Copy the task tag from
192 * the request so the completion checks pass.
193 */
194 resp_header->task_tag = tr->req->request_upiu.header.task_tag;
195
196 ufshci_req_queue_complete_tracker(tr);
197 }
198
199 void
ufshci_req_queue_fail(struct ufshci_controller * ctrlr,struct ufshci_req_queue * req_queue)200 ufshci_req_queue_fail(struct ufshci_controller *ctrlr,
201 struct ufshci_req_queue *req_queue)
202 {
203 struct ufshci_hw_queue *hwq = req_queue->qops.get_hw_queue(req_queue);
204 struct ufshci_tracker *tr;
205 int i;
206
207 if (!mtx_initialized(&hwq->qlock))
208 return;
209
210 mtx_lock(&hwq->qlock);
211
212 for (i = 0; i < req_queue->num_trackers; i++) {
213 tr = hwq->act_tr[i];
214
215 /*
216 * A slot in UFSHCI_SLOT_STATE_RESERVED is visible here
217 * only while its submit thread is failing a PRDT setup.
218 * That thread completes the request, so leave the slot
219 * alone.
220 */
221 if (tr->slot_state != UFSHCI_SLOT_STATE_SCHEDULED)
222 continue;
223
224 /*
225 * Claim the tracker under the lock. The completion
226 * scan only completes SCHEDULED slots, so it will
227 * skip this one while the lock is dropped.
228 */
229 tr->slot_state = UFSHCI_SLOT_STATE_NEED_ERROR_HANDLING;
230 mtx_unlock(&hwq->qlock);
231 ufshci_req_queue_manual_complete_tracker(tr,
232 UFSHCI_DESC_ABORTED,
233 UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);
234 mtx_lock(&hwq->qlock);
235 }
236
237 mtx_unlock(&hwq->qlock);
238 }
239
240 void
ufshci_req_queue_complete_tracker(struct ufshci_tracker * tr)241 ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr)
242 {
243 struct ufshci_req_queue *req_queue = tr->req_queue;
244 struct ufshci_hw_queue *hwq = tr->hwq;
245 struct ufshci_request *req = tr->req;
246 struct ufshci_completion cpl;
247 uint8_t ocs;
248 bool retry, error, retriable;
249
250 mtx_assert(&hwq->qlock, MA_NOTOWNED);
251
252 /* Copy the response from the Request Descriptor or UTP Command
253 * Descriptor. */
254 cpl.size = tr->response_size;
255 if (req_queue->is_task_mgmt) {
256 memcpy(&cpl.response_upiu,
257 (void *)hwq->utmrd[tr->slot_num].response_upiu, cpl.size);
258
259 ocs = hwq->utmrd[tr->slot_num].overall_command_status;
260 } else {
261 bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,
262 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
263
264 memcpy(&cpl.response_upiu, (void *)tr->ucd->response_upiu,
265 cpl.size);
266
267 ocs = hwq->utrd[tr->slot_num].overall_command_status;
268 }
269
270 error = ufshci_req_queue_response_is_error(req_queue, ocs,
271 &cpl.response_upiu);
272
273 /* Retry for admin commands. A failed controller must not retry. */
274 retriable = req->is_admin && !req_queue->ctrlr->is_failed;
275 retry = error && retriable &&
276 req->retries < req_queue->ctrlr->retry_count;
277 if (retry)
278 hwq->num_retries++;
279 if (error && req->retries >= req_queue->ctrlr->retry_count && retriable)
280 hwq->num_failures++;
281
282 KASSERT(tr->req, ("there is no request assigned to the tracker\n"));
283 KASSERT(cpl.response_upiu.header.task_tag ==
284 req->request_upiu.header.task_tag,
285 ("response task_tag does not match request task_tag\n"));
286
287 if (!retry) {
288 if (req->payload_valid) {
289 bus_dmamap_sync(req_queue->dma_tag_payload,
290 tr->payload_dma_map,
291 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
292 }
293 /* Copy response from the command descriptor */
294 if (req->cb_fn)
295 req->cb_fn(req->cb_arg, &cpl, error);
296 }
297
298 mtx_lock(&hwq->qlock);
299
300 /* Clear the UTRL Completion Notification register */
301 req_queue->qops.clear_cpl_ntf(req_queue->ctrlr, tr);
302
303 if (retry) {
304 req->retries++;
305 ufshci_req_queue_submit_tracker(req_queue, tr,
306 req->data_direction);
307 } else {
308 if (req->payload_valid) {
309 bus_dmamap_unload(req_queue->dma_tag_payload,
310 tr->payload_dma_map);
311 }
312
313 /* Clear tracker */
314 ufshci_free_request(req);
315 tr->req = NULL;
316 tr->slot_state = UFSHCI_SLOT_STATE_FREE;
317
318 TAILQ_REMOVE(&hwq->outstanding_tr, tr, tailq);
319 TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq);
320 }
321
322 mtx_unlock(&tr->hwq->qlock);
323 }
324
325 bool
ufshci_req_queue_process_completions(struct ufshci_req_queue * req_queue)326 ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue)
327 {
328 struct ufshci_hw_queue *hwq;
329 bool done;
330
331 hwq = req_queue->qops.get_hw_queue(req_queue);
332
333 mtx_lock(&hwq->recovery_lock);
334 done = req_queue->qops.process_cpl(req_queue);
335 mtx_unlock(&hwq->recovery_lock);
336
337 return (done);
338 }
339
340 static void
ufshci_payload_map(void * arg,bus_dma_segment_t * seg,int nseg,int error)341 ufshci_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
342 {
343 struct ufshci_tracker *tr = arg;
344 struct ufshci_prdt_entry *prdt_entry;
345 int i;
346
347 /*
348 * If the mapping operation failed, return immediately. The caller
349 * is responsible for detecting the error status and failing the
350 * tracker manually.
351 */
352 if (error != 0) {
353 ufshci_printf(tr->req_queue->ctrlr,
354 "Failed to map payload %d\n", error);
355 return;
356 }
357
358 prdt_entry = (struct ufshci_prdt_entry *)tr->ucd->prd_table;
359
360 tr->prdt_entry_cnt = nseg;
361
362 for (i = 0; i < nseg; i++) {
363 prdt_entry->data_base_address = htole64(seg[i].ds_addr) &
364 0xffffffff;
365 prdt_entry->data_base_address_upper = htole64(seg[i].ds_addr) >>
366 32;
367 prdt_entry->data_byte_count = htole32(seg[i].ds_len - 1);
368
369 ++prdt_entry;
370 }
371
372 bus_dmamap_sync(tr->req_queue->dma_tag_payload, tr->payload_dma_map,
373 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
374 }
375
376 static int
ufshci_req_queue_prepare_prdt(struct ufshci_tracker * tr)377 ufshci_req_queue_prepare_prdt(struct ufshci_tracker *tr)
378 {
379 struct ufshci_request *req = tr->req;
380 struct ufshci_utp_cmd_desc *cmd_desc = tr->ucd;
381 int error;
382
383 tr->prdt_off = UFSHCI_UTP_XFER_REQ_SIZE + UFSHCI_UTP_XFER_RESP_SIZE;
384
385 memset(cmd_desc->prd_table, 0, sizeof(cmd_desc->prd_table));
386
387 /* Filling PRDT enrties with payload */
388 error = bus_dmamap_load_mem(tr->req_queue->dma_tag_payload,
389 tr->payload_dma_map, &req->payload, ufshci_payload_map, tr,
390 BUS_DMA_NOWAIT);
391 if (error != 0) {
392 /*
393 * The dmamap operation failed, so we manually fail the
394 * tracker here with UFSHCI_DESC_INVALID_PRDT_ATTRIBUTES.
395 *
396 * ufshci_req_queue_manual_complete_tracker must not be called
397 * with the req_queue lock held.
398 */
399 ufshci_printf(tr->req_queue->ctrlr,
400 "bus_dmamap_load_mem returned with error:0x%x!\n", error);
401
402 mtx_unlock(&tr->hwq->qlock);
403 ufshci_req_queue_manual_complete_tracker(tr,
404 UFSHCI_DESC_INVALID_PRDT_ATTRIBUTES,
405 UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);
406 mtx_lock(&tr->hwq->qlock);
407 }
408
409 return (error);
410 }
411
412 static void
ufshci_req_queue_fill_utmr_descriptor(struct ufshci_utp_task_mgmt_req_desc * desc,struct ufshci_request * req)413 ufshci_req_queue_fill_utmr_descriptor(
414 struct ufshci_utp_task_mgmt_req_desc *desc, struct ufshci_request *req)
415 {
416 memset(desc, 0, sizeof(struct ufshci_utp_task_mgmt_req_desc));
417 desc->interrupt = true;
418 /* Set the initial value to Invalid. */
419 desc->overall_command_status = UFSHCI_UTMR_OCS_INVALID;
420
421 memcpy(desc->request_upiu, &req->request_upiu, req->request_size);
422 }
423
424 static void
ufshci_req_queue_fill_utr_descriptor(struct ufshci_utp_xfer_req_desc * desc,uint8_t data_direction,const uint64_t paddr,const uint16_t response_off,const uint16_t response_len,const uint16_t prdt_off,const uint16_t prdt_entry_cnt,const uint8_t total_ehs_length)425 ufshci_req_queue_fill_utr_descriptor(struct ufshci_utp_xfer_req_desc *desc,
426 uint8_t data_direction, const uint64_t paddr, const uint16_t response_off,
427 const uint16_t response_len, const uint16_t prdt_off,
428 const uint16_t prdt_entry_cnt, const uint8_t total_ehs_length)
429 {
430 uint8_t command_type;
431 /* Value to convert bytes to dwords */
432 const uint16_t dword_size = 4;
433
434 /*
435 * Set command type to UFS storage.
436 * The UFS 4.1 spec only defines 'UFS Storage' as a command type.
437 */
438 command_type = UFSHCI_COMMAND_TYPE_UFS_STORAGE;
439
440 memset(desc, 0, sizeof(struct ufshci_utp_xfer_req_desc));
441 desc->command_type = command_type;
442 desc->data_direction = data_direction;
443 desc->interrupt = true;
444 desc->total_ehs_length = total_ehs_length;
445 /* Set the initial value to Invalid. */
446 desc->overall_command_status = UFSHCI_UTR_OCS_INVALID;
447 desc->utp_command_descriptor_base_address = (uint32_t)(paddr &
448 0xffffffff);
449 desc->utp_command_descriptor_base_address_upper = (uint32_t)(paddr >>
450 32);
451
452 desc->response_upiu_offset = response_off / dword_size;
453 desc->response_upiu_length = response_len / dword_size;
454 desc->prdt_offset = prdt_off / dword_size;
455 desc->prdt_length = prdt_entry_cnt;
456 }
457
458 static void
ufshci_req_queue_timeout_recovery(struct ufshci_controller * ctrlr,struct ufshci_hw_queue * hwq)459 ufshci_req_queue_timeout_recovery(struct ufshci_controller *ctrlr,
460 struct ufshci_hw_queue *hwq)
461 {
462 /* TODO: Step 2. Logical unit reset */
463 /* TODO: Step 3. Target device reset */
464 /* TODO: Step 4. Bus reset */
465
466 /*
467 * Step 5. All previous commands were timeout.
468 * Recovery failed, reset the host controller.
469 */
470 ufshci_printf(ctrlr,
471 "Recovery step 5: Resetting controller due to a timeout.\n");
472 hwq->recovery_state = RECOVERY_WAITING;
473
474 ufshci_ctrlr_reset(ctrlr);
475 }
476
477 static void
ufshci_abort_complete(void * arg,const struct ufshci_completion * status,bool error)478 ufshci_abort_complete(void *arg, const struct ufshci_completion *status,
479 bool error)
480 {
481 struct ufshci_tracker *tr = arg;
482 uint32_t output_param1;
483
484 /*
485 * We still need to check the active tracker array, to cover race where
486 * I/O timed out at same time controller was completing the I/O. An
487 * abort request always is on the Task Management Request queue, but
488 * affects either an Task Management Request or an I/O (UTRL) queue, so
489 * take the appropriate queue lock for the original command's queue,
490 * since we'll need it to avoid races with the completion code and to
491 * complete the command manually.
492 */
493 mtx_lock(&tr->hwq->qlock);
494 if (tr->slot_state != UFSHCI_SLOT_STATE_FREE) {
495 mtx_unlock(&tr->hwq->qlock);
496 /*
497 * An I/O has timed out, and the controller was unable to abort
498 * it for some reason. And we've not processed a completion for
499 * it yet. Construct a fake completion status, and then complete
500 * the I/O's tracker manually.
501 */
502 ufshci_printf(tr->hwq->ctrlr,
503 "abort task request failed, aborting task manually\n");
504 ufshci_req_queue_manual_complete_tracker(tr,
505 UFSHCI_DESC_ABORTED, UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);
506
507 output_param1 = be32toh(
508 status->response_upiu.task_mgmt_response_upiu.output_param1);
509 if (output_param1 ==
510 UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_COMPLETE ||
511 output_param1 ==
512 UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_SUCCEEDED) {
513 ufshci_printf(tr->hwq->ctrlr,
514 "Warning: the abort task request completed \
515 successfully, but the original task is still incomplete.");
516 return;
517 }
518
519 /* Abort Task failed. Perform recovery steps 2-5 */
520 ufshci_req_queue_timeout_recovery(tr->hwq->ctrlr, tr->hwq);
521 } else {
522 mtx_unlock(&tr->hwq->qlock);
523 }
524 }
525
526 static void
ufshci_req_queue_timeout(void * arg)527 ufshci_req_queue_timeout(void *arg)
528 {
529 struct ufshci_hw_queue *hwq = arg;
530 struct ufshci_controller *ctrlr = hwq->ctrlr;
531 struct ufshci_tracker *tr;
532 sbintime_t now;
533 bool idle = true;
534 bool fast;
535
536 mtx_assert(&hwq->recovery_lock, MA_OWNED);
537
538 /*
539 * If the controller is failed, then stop polling. This ensures that any
540 * failure processing that races with the hwq timeout will fail safely.
541 */
542 if (ctrlr->is_failed) {
543 ufshci_printf(ctrlr,
544 "Failed controller, stopping watchdog timeout.\n");
545 hwq->timer_armed = false;
546 return;
547 }
548
549 /*
550 * Shutdown condition: We set hwq->timer_armed to false in
551 * ufshci_req_sdb_destroy before calling callout_drain. When we call
552 * that, this routine might get called one last time. Exit w/o setting a
553 * timeout. None of the watchdog stuff needs to be done since we're
554 * destroying the hwq.
555 */
556 if (!hwq->timer_armed) {
557 ufshci_printf(ctrlr,
558 "Timeout fired during ufshci_utr_req_queue_destroy\n");
559 return;
560 }
561
562 switch (hwq->recovery_state) {
563 case RECOVERY_NONE:
564 /*
565 * See if there's any recovery needed. First, do a fast check to
566 * see if anything could have timed out. If not, then skip
567 * everything else.
568 */
569 fast = false;
570 mtx_lock(&hwq->qlock);
571 now = getsbinuptime();
572 TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) {
573 /*
574 * If the first real transaction is not in timeout, then
575 * we're done. Otherwise, we try recovery.
576 */
577 idle = false;
578 if (now <= tr->deadline)
579 fast = true;
580 break;
581 }
582 mtx_unlock(&hwq->qlock);
583 if (idle || fast)
584 break;
585
586 /*
587 * There's a stale transaction at the start of the queue whose
588 * deadline has passed. Poll the competions as a last-ditch
589 * effort in case an interrupt has been missed.
590 */
591 hwq->req_queue->qops.process_cpl(hwq->req_queue);
592
593 /*
594 * Now that we've run the ISR, re-rheck to see if there's any
595 * timed out commands and abort them or reset the card if so.
596 */
597 mtx_lock(&hwq->qlock);
598 idle = true;
599 TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) {
600 /*
601 * If we know this tracker hasn't timed out, we also
602 * know all subsequent ones haven't timed out. The tr
603 * queue is in submission order and all normal commands
604 * in a queue have the same timeout (or the timeout was
605 * changed by the user, but we eventually timeout then).
606 */
607 idle = false;
608 if (now <= tr->deadline)
609 break;
610
611 /*
612 * Timeout recovery is performed in five steps. If
613 * recovery fails at any step, the process continues to
614 * the next one:
615 * next steps:
616 * Step 1. Abort task
617 * Step 2. Logical unit reset (TODO)
618 * Step 3. Target device reset (TODO)
619 * Step 4. Bus reset (TODO)
620 * Step 5. Host controller reset
621 *
622 * If the timeout occurred in the Task Management
623 * Request queue, ignore Step 1.
624 */
625 if (ctrlr->enable_aborts &&
626 !hwq->req_queue->is_task_mgmt &&
627 tr->req->cb_fn != ufshci_abort_complete) {
628 /*
629 * Step 1. Timeout expired, abort the task.
630 *
631 * This isn't an abort command, ask for a
632 * hardware abort. This goes to the Task
633 * Management Request queue which will reset the
634 * task if it times out.
635 */
636 ufshci_printf(ctrlr,
637 "Recovery step 1: Timeout occurred. aborting the task(%d).\n",
638 tr->req->request_upiu.header.task_tag);
639 if (ufshci_ctrlr_cmd_send_task_mgmt_request(ctrlr,
640 ufshci_abort_complete, tr,
641 UFSHCI_TASK_MGMT_FUNCTION_ABORT_TASK,
642 tr->req->request_upiu.header.lun,
643 tr->req->request_upiu.header.task_tag,
644 0) != 0) {
645 ufshci_req_queue_timeout_recovery(ctrlr,
646 hwq);
647 idle = false;
648 break;
649 }
650 } else {
651 /* Recovery Step 2-5 */
652 ufshci_req_queue_timeout_recovery(ctrlr, hwq);
653 idle = false;
654 break;
655 }
656 }
657 mtx_unlock(&hwq->qlock);
658 break;
659
660 case RECOVERY_WAITING:
661 /*
662 * These messages aren't interesting while we're suspended. We
663 * put the queues into waiting state while suspending.
664 * Suspending takes a while, so we'll see these during that time
665 * and they aren't diagnostic. At other times, they indicate a
666 * problem that's worth complaining about.
667 */
668 if (!device_is_suspended(ctrlr->dev))
669 ufshci_printf(ctrlr, "Waiting for reset to complete\n");
670 idle = false; /* We want to keep polling */
671 break;
672 }
673
674 /*
675 * Rearm the timeout.
676 */
677 if (!idle) {
678 callout_schedule_sbt(&hwq->timer, SBT_1S / 2, SBT_1S / 2, 0);
679 } else {
680 hwq->timer_armed = false;
681 }
682 }
683
684 /*
685 * Submit the tracker to the hardware.
686 */
687 static void
ufshci_req_queue_submit_tracker(struct ufshci_req_queue * req_queue,struct ufshci_tracker * tr,enum ufshci_data_direction data_direction)688 ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,
689 struct ufshci_tracker *tr, enum ufshci_data_direction data_direction)
690 {
691 struct ufshci_controller *ctrlr = req_queue->ctrlr;
692 struct ufshci_request *req = tr->req;
693 struct ufshci_hw_queue *hwq;
694 uint64_t ucd_paddr;
695 uint16_t request_len, response_off, response_len;
696 uint8_t slot_num = tr->slot_num;
697 int timeout;
698
699 hwq = req_queue->qops.get_hw_queue(req_queue);
700
701 mtx_assert(&hwq->qlock, MA_OWNED);
702
703 if (req->cb_fn == ufshci_completion_poll_cb)
704 timeout = 1;
705 else
706 timeout = ctrlr->timeout_period;
707 tr->deadline = getsbinuptime() + timeout * SBT_1S;
708 if (!hwq->timer_armed) {
709 hwq->timer_armed = true;
710 /*
711 * It wakes up once every 0.5 seconds to check if the deadline
712 * has passed.
713 */
714 callout_reset_sbt_on(&hwq->timer, SBT_1S / 2, SBT_1S / 2,
715 ufshci_req_queue_timeout, hwq, hwq->cpu, 0);
716 }
717
718 if (req_queue->is_task_mgmt) {
719 /* Prepare UTP Task Management Request Descriptor. */
720 ufshci_req_queue_fill_utmr_descriptor(&tr->hwq->utmrd[slot_num],
721 req);
722 } else {
723 KASSERT(req->request_size <= UFSHCI_UTP_XFER_REQ_SIZE &&
724 req->response_size <= UFSHCI_UTP_XFER_RESP_SIZE,
725 ("UPIU does not fit in the UTP command descriptor"));
726 request_len = req->request_size;
727 response_off = UFSHCI_UTP_XFER_REQ_SIZE;
728 response_len = req->response_size;
729
730 /* Prepare UTP Command Descriptor */
731 memcpy(tr->ucd, &req->request_upiu, request_len);
732 memset((uint8_t *)tr->ucd + response_off, 0, response_len);
733
734 /*
735 * Prepare PRDT. If the payload could not be mapped, the
736 * tracker has already been completed and released by the
737 * manual completion path, so the descriptor must not be
738 * built and the doorbell must not be rung.
739 */
740 if (req->payload_valid &&
741 ufshci_req_queue_prepare_prdt(tr) != 0)
742 return;
743
744 /* Prepare UTP Transfer Request Descriptor. */
745 ucd_paddr = tr->ucd_bus_addr;
746 ufshci_req_queue_fill_utr_descriptor(&tr->hwq->utrd[slot_num],
747 data_direction, ucd_paddr, response_off, response_len,
748 tr->prdt_off, tr->prdt_entry_cnt,
749 req->request_upiu.header.ehs_length);
750
751 bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,
752 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
753 }
754
755 bus_dmamap_sync(tr->hwq->dma_tag_queue, tr->hwq->queuemem_map,
756 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
757
758 tr->slot_state = UFSHCI_SLOT_STATE_SCHEDULED;
759
760 /* Ring the doorbell */
761 req_queue->qops.ring_doorbell(ctrlr, tr);
762 }
763
764 static int
_ufshci_req_queue_submit_request(struct ufshci_req_queue * req_queue,struct ufshci_request * req)765 _ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
766 struct ufshci_request *req)
767 {
768 struct ufshci_tracker *tr = NULL;
769 int error;
770
771 mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED);
772
773 if (req_queue->ctrlr->is_failed)
774 return (ENXIO);
775
776 error = req_queue->qops.reserve_slot(req_queue, &tr);
777 if (error != 0) {
778 ufshci_printf(req_queue->ctrlr, "Failed to get tracker");
779 return (error);
780 }
781 KASSERT(tr, ("There is no tracker allocated."));
782
783 if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED ||
784 tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED)
785 return (EBUSY);
786
787 /* Set the task_tag value to slot_num for traceability. */
788 req->request_upiu.header.task_tag = tr->slot_num;
789
790 tr->slot_state = UFSHCI_SLOT_STATE_RESERVED;
791 tr->response_size = req->response_size;
792 tr->deadline = SBT_MAX;
793 tr->req = req;
794
795 TAILQ_REMOVE(&tr->hwq->free_tr, tr, tailq);
796 TAILQ_INSERT_TAIL(&tr->hwq->outstanding_tr, tr, tailq);
797
798 ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction);
799
800 return (0);
801 }
802
803 int
ufshci_req_queue_submit_request(struct ufshci_req_queue * req_queue,struct ufshci_request * req)804 ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
805 struct ufshci_request *req)
806 {
807 struct ufshci_hw_queue *hwq;
808 uint32_t error;
809
810 /* TODO: MCQs should use a separate Admin queue. */
811
812 hwq = req_queue->qops.get_hw_queue(req_queue);
813 KASSERT(hwq, ("There is no HW queue allocated."));
814
815 mtx_lock(&hwq->qlock);
816 error = _ufshci_req_queue_submit_request(req_queue, req);
817 mtx_unlock(&hwq->qlock);
818
819 return (error);
820 }
821