1 /*-
2 * Copyright (c) 2025, Samsung Electronics Co., Ltd.
3 * Written by Jaeyoon Choi
4 *
5 * SPDX-License-Identifier: BSD-2-Clause
6 */
7
8 #include <sys/param.h>
9 #include <sys/bus.h>
10 #include <sys/conf.h>
11 #include <sys/domainset.h>
12 #include <sys/module.h>
13
14 #include "sys/kassert.h"
15 #include "ufshci_private.h"
16 #include "ufshci_reg.h"
17
18 static void
ufshci_req_sdb_cmd_desc_destroy(struct ufshci_req_queue * req_queue)19 ufshci_req_sdb_cmd_desc_destroy(struct ufshci_req_queue *req_queue)
20 {
21 struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
22 struct ufshci_tracker *tr;
23 int i;
24
25 for (i = 0; i < req_queue->num_trackers; i++) {
26 tr = hwq->act_tr[i];
27 bus_dmamap_destroy(req_queue->dma_tag_payload,
28 tr->payload_dma_map);
29 }
30
31 if (req_queue->ucd) {
32 bus_dmamap_unload(req_queue->dma_tag_ucd,
33 req_queue->ucdmem_map);
34 bus_dmamem_free(req_queue->dma_tag_ucd, req_queue->ucd,
35 req_queue->ucdmem_map);
36 req_queue->ucd = NULL;
37 }
38
39 if (req_queue->dma_tag_ucd) {
40 bus_dma_tag_destroy(req_queue->dma_tag_ucd);
41 req_queue->dma_tag_ucd = NULL;
42 }
43
44 free(req_queue->hwq->ucd_bus_addr, M_UFSHCI);
45 }
46
47 static void
ufshci_ucd_map(void * arg,bus_dma_segment_t * seg,int nseg,int error)48 ufshci_ucd_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
49 {
50 struct ufshci_hw_queue *hwq = arg;
51 int i;
52
53 if (error != 0) {
54 printf("ufshci: Failed to map UCD, error = %d\n", error);
55 return;
56 }
57
58 if (hwq->num_trackers != nseg) {
59 printf(
60 "ufshci: Failed to map UCD, num_trackers = %d, nseg = %d\n",
61 hwq->num_trackers, nseg);
62 return;
63 }
64
65 for (i = 0; i < nseg; i++) {
66 hwq->ucd_bus_addr[i] = seg[i].ds_addr;
67 }
68 }
69
70 static int
ufshci_req_sdb_cmd_desc_construct(struct ufshci_req_queue * req_queue,uint32_t num_entries,struct ufshci_controller * ctrlr)71 ufshci_req_sdb_cmd_desc_construct(struct ufshci_req_queue *req_queue,
72 uint32_t num_entries, struct ufshci_controller *ctrlr)
73 {
74 struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
75 size_t ucd_allocsz, payload_allocsz;
76 uint8_t *ucdmem;
77 int i, error;
78
79 req_queue->hwq->ucd_bus_addr = malloc(sizeof(bus_addr_t) *
80 req_queue->num_trackers,
81 M_UFSHCI, M_ZERO | M_NOWAIT);
82
83 /*
84 * Each component must be page aligned, and individual PRP lists
85 * cannot cross a page boundary.
86 */
87 ucd_allocsz = num_entries * sizeof(struct ufshci_utp_cmd_desc);
88 ucd_allocsz = roundup2(ucd_allocsz, ctrlr->page_size);
89 payload_allocsz = num_entries * ctrlr->max_xfer_size;
90
91 /*
92 * Allocate physical memory for UTP Command Descriptor (UCD)
93 * Note: UFSHCI UCD format is restricted to 128-byte alignment.
94 */
95 error = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 128, 0,
96 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, ucd_allocsz,
97 howmany(ucd_allocsz, sizeof(struct ufshci_utp_cmd_desc)),
98 sizeof(struct ufshci_utp_cmd_desc), 0, NULL, NULL,
99 &req_queue->dma_tag_ucd);
100 if (error != 0) {
101 ufshci_printf(ctrlr, "request cmd desc tag create failed %d\n",
102 error);
103 goto out;
104 }
105
106 if (bus_dmamem_alloc(req_queue->dma_tag_ucd, (void **)&ucdmem,
107 BUS_DMA_COHERENT | BUS_DMA_NOWAIT, &req_queue->ucdmem_map)) {
108 ufshci_printf(ctrlr, "failed to allocate cmd desc memory\n");
109 goto out;
110 }
111
112 if (bus_dmamap_load(req_queue->dma_tag_ucd, req_queue->ucdmem_map,
113 ucdmem, ucd_allocsz, ufshci_ucd_map, hwq, 0) != 0) {
114 ufshci_printf(ctrlr, "failed to load cmd desc memory\n");
115 bus_dmamem_free(req_queue->dma_tag_ucd, req_queue->ucd,
116 req_queue->ucdmem_map);
117 goto out;
118 }
119
120 req_queue->ucd = (struct ufshci_utp_cmd_desc *)ucdmem;
121
122 /*
123 * Allocate physical memory for PRDT
124 * Note: UFSHCI PRDT format is restricted to 8-byte alignment.
125 */
126 error = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 8,
127 ctrlr->page_size, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
128 payload_allocsz, howmany(payload_allocsz, ctrlr->page_size) + 1,
129 ctrlr->page_size, 0, NULL, NULL, &req_queue->dma_tag_payload);
130 if (error != 0) {
131 ufshci_printf(ctrlr, "request prdt tag create failed %d\n",
132 error);
133 goto out;
134 }
135
136 for (i = 0; i < req_queue->num_trackers; i++) {
137 bus_dmamap_create(req_queue->dma_tag_payload, 0,
138 &hwq->act_tr[i]->payload_dma_map);
139
140 hwq->act_tr[i]->ucd = (struct ufshci_utp_cmd_desc *)ucdmem;
141 hwq->act_tr[i]->ucd_bus_addr = hwq->ucd_bus_addr[i];
142
143 ucdmem += sizeof(struct ufshci_utp_cmd_desc);
144 }
145
146 return (0);
147 out:
148 ufshci_req_sdb_cmd_desc_destroy(req_queue);
149 return (ENOMEM);
150 }
151
152 int
ufshci_req_sdb_construct(struct ufshci_controller * ctrlr,struct ufshci_req_queue * req_queue,uint32_t num_entries,bool is_task_mgmt)153 ufshci_req_sdb_construct(struct ufshci_controller *ctrlr,
154 struct ufshci_req_queue *req_queue, uint32_t num_entries, bool is_task_mgmt)
155 {
156 struct ufshci_hw_queue *hwq;
157 size_t desc_size, alloc_size;
158 uint64_t queuemem_phys;
159 uint8_t *queuemem;
160 struct ufshci_tracker *tr;
161 const size_t lock_name_len = 32;
162 char qlock_name[lock_name_len], recovery_lock_name[lock_name_len];
163 char *base;
164 int i, error;
165
166 req_queue->ctrlr = ctrlr;
167 req_queue->is_task_mgmt = is_task_mgmt;
168 req_queue->num_entries = num_entries;
169 /*
170 * In Single Doorbell mode, the number of queue entries and the number
171 * of trackers are the same.
172 */
173 req_queue->num_trackers = num_entries;
174
175 /* Single Doorbell mode uses only one queue. (UFSHCI_SDB_Q = 0) */
176 req_queue->hwq = malloc(sizeof(struct ufshci_hw_queue), M_UFSHCI,
177 M_ZERO | M_NOWAIT);
178 hwq = &req_queue->hwq[UFSHCI_SDB_Q];
179 hwq->num_entries = req_queue->num_entries;
180 hwq->num_trackers = req_queue->num_trackers;
181 hwq->ctrlr = ctrlr;
182 hwq->req_queue = req_queue;
183
184 base = is_task_mgmt ? "ufshci utmrq" : "ufshci utrq";
185 snprintf(qlock_name, sizeof(qlock_name), "%s #%d lock", base,
186 UFSHCI_SDB_Q);
187 snprintf(recovery_lock_name, sizeof(recovery_lock_name),
188 "%s #%d recovery lock", base, UFSHCI_SDB_Q);
189
190 mtx_init(&hwq->qlock, qlock_name, NULL, MTX_DEF);
191 mtx_init(&hwq->recovery_lock, recovery_lock_name, NULL, MTX_DEF);
192
193 callout_init_mtx(&hwq->timer, &hwq->recovery_lock, 0);
194 hwq->timer_armed = false;
195 hwq->recovery_state = RECOVERY_WAITING;
196
197 /*
198 * Allocate physical memory for request queue (UTP Transfer Request
199 * Descriptor (UTRD) or UTP Task Management Request Descriptor (UTMRD))
200 * Note: UTRD/UTMRD format is restricted to 1024-byte alignment.
201 */
202 desc_size = is_task_mgmt ?
203 sizeof(struct ufshci_utp_task_mgmt_req_desc) :
204 sizeof(struct ufshci_utp_xfer_req_desc);
205 alloc_size = num_entries * desc_size;
206 error = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 1024,
207 ctrlr->page_size, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
208 alloc_size, 1, alloc_size, 0, NULL, NULL, &hwq->dma_tag_queue);
209 if (error != 0) {
210 ufshci_printf(ctrlr, "request queue tag create failed %d\n",
211 error);
212 goto out;
213 }
214
215 if (bus_dmamem_alloc(hwq->dma_tag_queue, (void **)&queuemem,
216 BUS_DMA_COHERENT | BUS_DMA_NOWAIT, &hwq->queuemem_map)) {
217 ufshci_printf(ctrlr,
218 "failed to allocate request queue memory\n");
219 goto out;
220 }
221
222 if (bus_dmamap_load(hwq->dma_tag_queue, hwq->queuemem_map, queuemem,
223 alloc_size, ufshci_single_map, &queuemem_phys, 0) != 0) {
224 ufshci_printf(ctrlr, "failed to load request queue memory\n");
225 bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd,
226 hwq->queuemem_map);
227 goto out;
228 }
229
230 hwq->num_cmds = 0;
231 hwq->num_intr_handler_calls = 0;
232 hwq->num_retries = 0;
233 hwq->num_failures = 0;
234 hwq->req_queue_addr = queuemem_phys;
235
236 /* Allocate trackers */
237 hwq->act_tr = malloc_domainset(sizeof(struct ufshci_tracker *) *
238 req_queue->num_entries,
239 M_UFSHCI, DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK);
240
241 TAILQ_INIT(&hwq->free_tr);
242 TAILQ_INIT(&hwq->outstanding_tr);
243
244 for (i = 0; i < req_queue->num_trackers; i++) {
245 tr = malloc_domainset(sizeof(struct ufshci_tracker), M_UFSHCI,
246 DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK);
247
248 tr->req_queue = req_queue;
249 tr->slot_num = i;
250 tr->slot_state = UFSHCI_SLOT_STATE_FREE;
251 TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq);
252
253 hwq->act_tr[i] = tr;
254 }
255
256 if (is_task_mgmt) {
257 /* UTP Task Management Request (UTMR) */
258 uint32_t utmrlba, utmrlbau;
259
260 hwq->utmrd = (struct ufshci_utp_task_mgmt_req_desc *)queuemem;
261
262 utmrlba = hwq->req_queue_addr & 0xffffffff;
263 utmrlbau = hwq->req_queue_addr >> 32;
264 ufshci_mmio_write_4(ctrlr, utmrlba, utmrlba);
265 ufshci_mmio_write_4(ctrlr, utmrlbau, utmrlbau);
266 } else {
267 /* UTP Transfer Request (UTR) */
268 uint32_t utrlba, utrlbau;
269
270 hwq->utrd = (struct ufshci_utp_xfer_req_desc *)queuemem;
271
272 /*
273 * Allocate physical memory for the command descriptor.
274 * UTP Transfer Request (UTR) requires memory for a separate
275 * command in addition to the queue.
276 */
277 if (ufshci_req_sdb_cmd_desc_construct(req_queue, num_entries,
278 ctrlr) != 0) {
279 ufshci_printf(ctrlr,
280 "failed to construct cmd descriptor memory\n");
281 goto out;
282 }
283
284 utrlba = hwq->req_queue_addr & 0xffffffff;
285 utrlbau = hwq->req_queue_addr >> 32;
286 ufshci_mmio_write_4(ctrlr, utrlba, utrlba);
287 ufshci_mmio_write_4(ctrlr, utrlbau, utrlbau);
288 }
289
290 return (0);
291 out:
292 ufshci_req_sdb_destroy(ctrlr, req_queue);
293 return (ENOMEM);
294 }
295
296 void
ufshci_req_sdb_destroy(struct ufshci_controller * ctrlr,struct ufshci_req_queue * req_queue)297 ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr,
298 struct ufshci_req_queue *req_queue)
299 {
300 struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
301 struct ufshci_tracker *tr;
302 int i;
303
304 mtx_lock(&hwq->recovery_lock);
305 hwq->timer_armed = false;
306 mtx_unlock(&hwq->recovery_lock);
307 callout_drain(&hwq->timer);
308
309 if (!req_queue->is_task_mgmt)
310 ufshci_req_sdb_cmd_desc_destroy(&ctrlr->transfer_req_queue);
311
312 for (i = 0; i < req_queue->num_trackers; i++) {
313 tr = hwq->act_tr[i];
314 free(tr, M_UFSHCI);
315 }
316
317 if (hwq->act_tr) {
318 free(hwq->act_tr, M_UFSHCI);
319 hwq->act_tr = NULL;
320 }
321
322 if (hwq->utrd != NULL) {
323 bus_dmamap_unload(hwq->dma_tag_queue, hwq->queuemem_map);
324 bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd,
325 hwq->queuemem_map);
326 hwq->utrd = NULL;
327 }
328
329 if (hwq->dma_tag_queue) {
330 bus_dma_tag_destroy(hwq->dma_tag_queue);
331 hwq->dma_tag_queue = NULL;
332 }
333
334 if (mtx_initialized(&hwq->recovery_lock))
335 mtx_destroy(&hwq->recovery_lock);
336 if (mtx_initialized(&hwq->qlock))
337 mtx_destroy(&hwq->qlock);
338
339 free(req_queue->hwq, M_UFSHCI);
340 }
341
342 struct ufshci_hw_queue *
ufshci_req_sdb_get_hw_queue(struct ufshci_req_queue * req_queue)343 ufshci_req_sdb_get_hw_queue(struct ufshci_req_queue *req_queue)
344 {
345 return &req_queue->hwq[UFSHCI_SDB_Q];
346 }
347
348 void
ufshci_req_sdb_disable(struct ufshci_controller * ctrlr,struct ufshci_req_queue * req_queue)349 ufshci_req_sdb_disable(struct ufshci_controller *ctrlr,
350 struct ufshci_req_queue *req_queue)
351 {
352 struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
353 struct ufshci_tracker *tr, *tr_temp;
354
355 mtx_lock(&hwq->recovery_lock);
356 mtx_lock(&hwq->qlock);
357
358 if (mtx_initialized(&hwq->recovery_lock))
359 mtx_assert(&hwq->recovery_lock, MA_OWNED);
360 if (mtx_initialized(&hwq->qlock))
361 mtx_assert(&hwq->qlock, MA_OWNED);
362
363 hwq->recovery_state = RECOVERY_WAITING;
364 TAILQ_FOREACH_SAFE(tr, &hwq->outstanding_tr, tailq, tr_temp) {
365 tr->deadline = SBT_MAX;
366 }
367
368 mtx_unlock(&hwq->qlock);
369 mtx_unlock(&hwq->recovery_lock);
370 }
371
372 int
ufshci_req_sdb_enable(struct ufshci_controller * ctrlr,struct ufshci_req_queue * req_queue)373 ufshci_req_sdb_enable(struct ufshci_controller *ctrlr,
374 struct ufshci_req_queue *req_queue)
375 {
376 struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
377 int error = 0;
378
379 mtx_lock(&hwq->recovery_lock);
380 mtx_lock(&hwq->qlock);
381
382 if (req_queue->is_task_mgmt) {
383 uint32_t hcs, utmrldbr, utmrlrsr;
384 uint32_t utmrlba, utmrlbau;
385
386 /*
387 * Some controllers require re-enabling. When a controller is
388 * re-enabled, the utmrlba registers are initialized, and these
389 * must be reconfigured upon re-enabling.
390 */
391 utmrlba = hwq->req_queue_addr & 0xffffffff;
392 utmrlbau = hwq->req_queue_addr >> 32;
393 ufshci_mmio_write_4(ctrlr, utmrlba, utmrlba);
394 ufshci_mmio_write_4(ctrlr, utmrlbau, utmrlbau);
395
396 hcs = ufshci_mmio_read_4(ctrlr, hcs);
397 if (!(hcs & UFSHCIM(UFSHCI_HCS_REG_UTMRLRDY))) {
398 ufshci_printf(ctrlr,
399 "UTP task management request list is not ready\n");
400 error = ENXIO;
401 goto out;
402 }
403
404 utmrldbr = ufshci_mmio_read_4(ctrlr, utmrldbr);
405 if (utmrldbr != 0) {
406 ufshci_printf(ctrlr,
407 "UTP task management request list door bell is not ready\n");
408 error = ENXIO;
409 goto out;
410 }
411
412 utmrlrsr = UFSHCIM(UFSHCI_UTMRLRSR_REG_UTMRLRSR);
413 ufshci_mmio_write_4(ctrlr, utmrlrsr, utmrlrsr);
414 } else {
415 uint32_t hcs, utrldbr, utrlcnr, utrlrsr;
416 uint32_t utrlba, utrlbau;
417
418 /*
419 * Some controllers require re-enabling. When a controller is
420 * re-enabled, the utrlba registers are initialized, and these
421 * must be reconfigured upon re-enabling.
422 */
423 utrlba = hwq->req_queue_addr & 0xffffffff;
424 utrlbau = hwq->req_queue_addr >> 32;
425 ufshci_mmio_write_4(ctrlr, utrlba, utrlba);
426 ufshci_mmio_write_4(ctrlr, utrlbau, utrlbau);
427
428 hcs = ufshci_mmio_read_4(ctrlr, hcs);
429 if (!(hcs & UFSHCIM(UFSHCI_HCS_REG_UTRLRDY))) {
430 ufshci_printf(ctrlr,
431 "UTP transfer request list is not ready\n");
432 error = ENXIO;
433 goto out;
434 }
435
436 utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr);
437 if (utrldbr != 0) {
438 ufshci_printf(ctrlr,
439 "UTP transfer request list door bell is not ready\n");
440 ufshci_printf(ctrlr,
441 "Clear the UTP transfer request list door bell\n");
442 ufshci_mmio_write_4(ctrlr, utrldbr, utrldbr);
443 }
444
445 utrlcnr = ufshci_mmio_read_4(ctrlr, utrlcnr);
446 if (utrlcnr != 0) {
447 ufshci_printf(ctrlr,
448 "UTP transfer request list notification is not ready\n");
449 ufshci_printf(ctrlr,
450 "Clear the UTP transfer request list notification\n");
451 ufshci_mmio_write_4(ctrlr, utrlcnr, utrlcnr);
452 }
453
454 utrlrsr = UFSHCIM(UFSHCI_UTRLRSR_REG_UTRLRSR);
455 ufshci_mmio_write_4(ctrlr, utrlrsr, utrlrsr);
456 }
457
458 if (mtx_initialized(&hwq->recovery_lock))
459 mtx_assert(&hwq->recovery_lock, MA_OWNED);
460 if (mtx_initialized(&hwq->qlock))
461 mtx_assert(&hwq->qlock, MA_OWNED);
462 KASSERT(!req_queue->ctrlr->is_failed, ("Enabling a failed hwq\n"));
463
464 hwq->recovery_state = RECOVERY_NONE;
465
466 out:
467 mtx_unlock(&hwq->qlock);
468 mtx_unlock(&hwq->recovery_lock);
469 return (error);
470 }
471
472 int
ufshci_req_sdb_reserve_slot(struct ufshci_req_queue * req_queue,struct ufshci_tracker ** tr)473 ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue,
474 struct ufshci_tracker **tr)
475 {
476 struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
477 uint8_t i;
478
479 for (i = 0; i < req_queue->num_entries; i++) {
480 if (hwq->act_tr[i]->slot_state == UFSHCI_SLOT_STATE_FREE) {
481 *tr = hwq->act_tr[i];
482 (*tr)->hwq = hwq;
483 return (0);
484 }
485 }
486 return (EBUSY);
487 }
488
489 void
ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller * ctrlr,struct ufshci_tracker * tr)490 ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr,
491 struct ufshci_tracker *tr)
492 {
493 /*
494 * NOP
495 * UTP Task Management does not have a Completion Notification
496 * Register.
497 */
498 }
499
500 void
ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller * ctrlr,struct ufshci_tracker * tr)501 ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller *ctrlr,
502 struct ufshci_tracker *tr)
503 {
504 uint32_t utrlcnr;
505
506 utrlcnr = 1 << tr->slot_num;
507 ufshci_mmio_write_4(ctrlr, utrlcnr, utrlcnr);
508 }
509
510 void
ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller * ctrlr,struct ufshci_tracker * tr)511 ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr,
512 struct ufshci_tracker *tr)
513 {
514 uint32_t utmrldbr = 0;
515
516 utmrldbr |= 1 << tr->slot_num;
517 ufshci_mmio_write_4(ctrlr, utmrldbr, utmrldbr);
518
519 tr->req_queue->hwq[UFSHCI_SDB_Q].num_cmds++;
520 }
521
522 void
ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller * ctrlr,struct ufshci_tracker * tr)523 ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr,
524 struct ufshci_tracker *tr)
525 {
526 uint32_t utrldbr = 0;
527
528 utrldbr |= 1 << tr->slot_num;
529 ufshci_mmio_write_4(ctrlr, utrldbr, utrldbr);
530
531 tr->req_queue->hwq[UFSHCI_SDB_Q].num_cmds++;
532 }
533
534 bool
ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller * ctrlr,uint8_t slot)535 ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
536 uint8_t slot)
537 {
538 uint32_t utmrldbr;
539
540 utmrldbr = ufshci_mmio_read_4(ctrlr, utmrldbr);
541 return (!(utmrldbr & (1 << slot)));
542 }
543
544 bool
ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller * ctrlr,uint8_t slot)545 ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
546 uint8_t slot)
547 {
548 uint32_t utrldbr;
549
550 utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr);
551 return (!(utrldbr & (1 << slot)));
552 }
553
554 bool
ufshci_req_sdb_process_cpl(struct ufshci_req_queue * req_queue)555 ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue)
556 {
557 struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
558 struct ufshci_tracker *tr;
559 uint8_t slot;
560 bool done = false;
561
562 mtx_assert(&hwq->recovery_lock, MA_OWNED);
563
564 hwq->num_intr_handler_calls++;
565
566 bus_dmamap_sync(hwq->dma_tag_queue, hwq->queuemem_map,
567 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
568
569 for (slot = 0; slot < req_queue->num_entries; slot++) {
570 tr = hwq->act_tr[slot];
571
572 KASSERT(tr, ("there is no tracker assigned to the slot"));
573 /*
574 * When the response is delivered from the device, the doorbell
575 * is cleared.
576 */
577 if (tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED &&
578 req_queue->qops.is_doorbell_cleared(req_queue->ctrlr,
579 slot)) {
580 ufshci_req_queue_complete_tracker(tr);
581 done = true;
582 }
583 }
584
585 return (done);
586 }
587
588 int
ufshci_req_sdb_get_inflight_io(struct ufshci_controller * ctrlr)589 ufshci_req_sdb_get_inflight_io(struct ufshci_controller *ctrlr)
590 {
591 /* TODO: Implement inflight io*/
592
593 return (0);
594 }
595