1 /*-
2 * Copyright (c) 2025, Samsung Electronics Co., Ltd.
3 * Written by Jaeyoon Choi
4 *
5 * SPDX-License-Identifier: BSD-2-Clause
6 */
7
8 #include <sys/param.h>
9 #include <sys/bus.h>
10 #include <sys/conf.h>
11
12 #include "ufshci_private.h"
13 #include "ufshci_reg.h"
14
15 static void
ufshci_ctrlr_fail(struct ufshci_controller * ctrlr)16 ufshci_ctrlr_fail(struct ufshci_controller *ctrlr)
17 {
18 ctrlr->is_failed = true;
19
20 ufshci_req_queue_fail(ctrlr, &ctrlr->task_mgmt_req_queue);
21 ufshci_req_queue_fail(ctrlr, &ctrlr->transfer_req_queue);
22 }
23
24 /* Some controllers require a reinit after switching to the max gear. */
25 static int
ufshci_ctrlr_reinit_after_max_gear_switch(struct ufshci_controller * ctrlr)26 ufshci_ctrlr_reinit_after_max_gear_switch(struct ufshci_controller *ctrlr)
27 {
28 int error;
29
30 /* Reset device */
31 ufshci_utmr_req_queue_disable(ctrlr);
32 ufshci_utr_req_queue_disable(ctrlr);
33
34 error = ufshci_ctrlr_disable(ctrlr);
35 if (error != 0)
36 return (error);
37
38 error = ufshci_ctrlr_enable(ctrlr);
39 if (error != 0)
40 return (error);
41
42 error = ufshci_utmr_req_queue_enable(ctrlr);
43 if (error != 0)
44 return (error);
45
46 error = ufshci_utr_req_queue_enable(ctrlr);
47 if (error != 0)
48 return (error);
49
50 error = ufshci_ctrlr_send_nop(ctrlr);
51 if (error != 0)
52 return (error);
53
54 /* Reinit the target device. */
55 error = ufshci_dev_init(ctrlr);
56 if (error != 0)
57 return (error);
58
59 /* Initialize Reference Clock */
60 error = ufshci_dev_init_reference_clock(ctrlr);
61 if (error != 0)
62 return (error);
63
64 /* Initialize unipro */
65 return (ufshci_dev_init_unipro(ctrlr));
66 }
67
68 static void
ufshci_ctrlr_start(struct ufshci_controller * ctrlr,bool resetting)69 ufshci_ctrlr_start(struct ufshci_controller *ctrlr, bool resetting)
70 {
71 TSENTER();
72
73 /*
74 * If `resetting` is true, we are on the reset path.
75 * Re-enable request queues here because ufshci_ctrlr_reset_task()
76 * disables them during reset.
77 */
78 if (resetting) {
79 if (ufshci_utmr_req_queue_enable(ctrlr) != 0) {
80 ufshci_ctrlr_fail(ctrlr);
81 return;
82 }
83 if (ufshci_utr_req_queue_enable(ctrlr) != 0) {
84 ufshci_ctrlr_fail(ctrlr);
85 return;
86 }
87 }
88
89 if (ufshci_ctrlr_send_nop(ctrlr) != 0) {
90 ufshci_ctrlr_fail(ctrlr);
91 return;
92 }
93
94 /* Initialize UFS target drvice */
95 if (ufshci_dev_init(ctrlr) != 0) {
96 ufshci_ctrlr_fail(ctrlr);
97 return;
98 }
99
100 /* Initialize Reference Clock */
101 if (ufshci_dev_init_reference_clock(ctrlr) != 0) {
102 ufshci_ctrlr_fail(ctrlr);
103 return;
104 }
105
106 /* Initialize unipro */
107 if (ufshci_dev_init_unipro(ctrlr) != 0) {
108 ufshci_ctrlr_fail(ctrlr);
109 return;
110 }
111
112 /*
113 * Initialize UIC Power Mode
114 * QEMU UFS devices do not support unipro and power mode.
115 */
116 if (!(ctrlr->quirks & UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE) &&
117 ufshci_dev_init_uic_power_mode(ctrlr) != 0) {
118 ufshci_ctrlr_fail(ctrlr);
119 return;
120 }
121
122 ufshci_dev_init_uic_link_state(ctrlr);
123
124 if ((ctrlr->quirks & UFSHCI_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) &&
125 ufshci_ctrlr_reinit_after_max_gear_switch(ctrlr) != 0) {
126 ufshci_ctrlr_fail(ctrlr);
127 return;
128 }
129
130 /* Read Controller Descriptor (Device, Geometry) */
131 if (ufshci_dev_get_descriptor(ctrlr) != 0) {
132 ufshci_ctrlr_fail(ctrlr);
133 return;
134 }
135
136 if (ufshci_dev_config_write_booster(ctrlr)) {
137 ufshci_ctrlr_fail(ctrlr);
138 return;
139 }
140
141 ufshci_dev_init_auto_hibernate(ctrlr);
142
143 /* TODO: Configure Write Protect */
144
145 /* TODO: Configure Background Operations */
146
147 /*
148 * If the reset is due to a timeout, it is already attached to the SIM
149 * and does not need to be attached again.
150 */
151 if (!resetting && ufshci_sim_attach(ctrlr) != 0) {
152 ufshci_ctrlr_fail(ctrlr);
153 return;
154 }
155
156 /* Initialize UFS Power Mode */
157 if (ufshci_dev_init_ufs_power_mode(ctrlr) != 0) {
158 ufshci_ctrlr_fail(ctrlr);
159 return;
160 }
161
162 TSEXIT();
163 }
164
165 static int
ufshci_ctrlr_disable_host_ctrlr(struct ufshci_controller * ctrlr)166 ufshci_ctrlr_disable_host_ctrlr(struct ufshci_controller *ctrlr)
167 {
168 int timeout = ticks + MSEC_2_TICKS(ctrlr->device_init_timeout_in_ms);
169 sbintime_t delta_t = SBT_1US;
170 uint32_t hce;
171
172 hce = ufshci_mmio_read_4(ctrlr, hce);
173
174 /* If UFS host controller is already enabled, disable it. */
175 if (UFSHCIV(UFSHCI_HCE_REG_HCE, hce)) {
176 hce &= ~UFSHCIM(UFSHCI_HCE_REG_HCE);
177 ufshci_mmio_write_4(ctrlr, hce, hce);
178 }
179
180 /* Wait for the HCE flag to change */
181 while (1) {
182 hce = ufshci_mmio_read_4(ctrlr, hce);
183 if (!UFSHCIV(UFSHCI_HCE_REG_HCE, hce))
184 break;
185 if (timeout - ticks < 0) {
186 ufshci_printf(ctrlr,
187 "host controller failed to disable "
188 "within %d ms\n",
189 ctrlr->device_init_timeout_in_ms);
190 return (ENXIO);
191 }
192
193 pause_sbt("ufshci_disable_hce", delta_t, 0, C_PREL(1));
194 delta_t = min(SBT_1MS, delta_t * 3 / 2);
195 }
196
197 return (0);
198 }
199
200 static int
ufshci_ctrlr_enable_host_ctrlr(struct ufshci_controller * ctrlr)201 ufshci_ctrlr_enable_host_ctrlr(struct ufshci_controller *ctrlr)
202 {
203 int timeout = ticks + MSEC_2_TICKS(ctrlr->device_init_timeout_in_ms);
204 sbintime_t delta_t = SBT_1US;
205 uint32_t hce;
206
207 hce = ufshci_mmio_read_4(ctrlr, hce);
208
209 /* Enable UFS host controller */
210 hce |= UFSHCIM(UFSHCI_HCE_REG_HCE);
211 ufshci_mmio_write_4(ctrlr, hce, hce);
212
213 /*
214 * During the controller initialization, the value of the HCE bit is
215 * unstable, so we need to read the HCE value after some time after
216 * initialization is complete.
217 */
218 pause_sbt("ufshci_enable_hce", ustosbt(100), 0, C_PREL(1));
219
220 /* Wait for the HCE flag to change */
221 while (1) {
222 hce = ufshci_mmio_read_4(ctrlr, hce);
223 if (UFSHCIV(UFSHCI_HCE_REG_HCE, hce))
224 break;
225 if (timeout - ticks < 0) {
226 ufshci_printf(ctrlr,
227 "host controller failed to enable "
228 "within %d ms\n",
229 ctrlr->device_init_timeout_in_ms);
230 return (ENXIO);
231 }
232
233 pause_sbt("ufshci_enable_hce", delta_t, 0, C_PREL(1));
234 delta_t = min(SBT_1MS, delta_t * 3 / 2);
235 }
236
237 return (0);
238 }
239
240 int
ufshci_ctrlr_disable(struct ufshci_controller * ctrlr)241 ufshci_ctrlr_disable(struct ufshci_controller *ctrlr)
242 {
243 int error;
244
245 /* Disable all interrupts */
246 ufshci_mmio_write_4(ctrlr, ie, 0);
247
248 error = ufshci_ctrlr_disable_host_ctrlr(ctrlr);
249 return (error);
250 }
251
252 int
ufshci_ctrlr_enable(struct ufshci_controller * ctrlr)253 ufshci_ctrlr_enable(struct ufshci_controller *ctrlr)
254 {
255 uint32_t ie, hcs;
256 int error;
257
258 error = ufshci_ctrlr_enable_host_ctrlr(ctrlr);
259 if (error)
260 return (error);
261
262 /* Send DME_LINKSTARTUP command to start the link startup procedure */
263 error = ufshci_uic_send_dme_link_startup(ctrlr);
264 if (error)
265 return (error);
266
267 /*
268 * The device_present(UFSHCI_HCS_REG_DP) bit becomes true if the host
269 * controller has successfully received a Link Startup UIC command
270 * response and the UFS device has found a physical link to the
271 * controller.
272 */
273 hcs = ufshci_mmio_read_4(ctrlr, hcs);
274 if (!UFSHCIV(UFSHCI_HCS_REG_DP, hcs)) {
275 ufshci_printf(ctrlr, "UFS device not found\n");
276 return (ENXIO);
277 }
278
279 /* Enable additional interrupts by programming the IE register. */
280 ie = ufshci_mmio_read_4(ctrlr, ie);
281 ie |= UFSHCIM(UFSHCI_IE_REG_UTRCE); /* UTR Completion */
282 ie |= UFSHCIM(UFSHCI_IE_REG_UEE); /* UIC Error */
283 ie |= UFSHCIM(UFSHCI_IE_REG_UTMRCE); /* UTMR Completion */
284 ie |= UFSHCIM(UFSHCI_IE_REG_DFEE); /* Device Fatal Error */
285 ie |= UFSHCIM(UFSHCI_IE_REG_UTPEE); /* UTP Error */
286 ie |= UFSHCIM(UFSHCI_IE_REG_HCFEE); /* Host Ctrlr Fatal Error */
287 ie |= UFSHCIM(UFSHCI_IE_REG_SBFEE); /* System Bus Fatal Error */
288 ie |= UFSHCIM(UFSHCI_IE_REG_CEFEE); /* Crypto Engine Fatal Error */
289 ufshci_mmio_write_4(ctrlr, ie, ie);
290
291 /* TODO: Initialize interrupt Aggregation Control Register (UTRIACR) */
292
293 return (0);
294 }
295
296 static int
ufshci_ctrlr_hw_reset(struct ufshci_controller * ctrlr)297 ufshci_ctrlr_hw_reset(struct ufshci_controller *ctrlr)
298 {
299 int error;
300
301 error = ufshci_ctrlr_disable(ctrlr);
302 if (error)
303 return (error);
304
305 error = ufshci_ctrlr_enable(ctrlr);
306 return (error);
307 }
308
309 static void
ufshci_ctrlr_reset_task(void * arg,int pending)310 ufshci_ctrlr_reset_task(void *arg, int pending)
311 {
312 struct ufshci_controller *ctrlr = arg;
313 int error;
314
315 /* Release resources */
316 ufshci_utmr_req_queue_disable(ctrlr);
317 ufshci_utr_req_queue_disable(ctrlr);
318
319 error = ufshci_ctrlr_hw_reset(ctrlr);
320 if (error)
321 return (ufshci_ctrlr_fail(ctrlr));
322
323 ufshci_ctrlr_start(ctrlr, true);
324 }
325
326 int
ufshci_ctrlr_construct(struct ufshci_controller * ctrlr,device_t dev)327 ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev)
328 {
329 uint32_t ver, cap, ahit;
330 uint32_t timeout_period, retry_count;
331 int error;
332
333 ctrlr->device_init_timeout_in_ms = UFSHCI_DEVICE_INIT_TIMEOUT_MS;
334 ctrlr->uic_cmd_timeout_in_ms = UFSHCI_UIC_CMD_TIMEOUT_MS;
335 ctrlr->dev = dev;
336 ctrlr->sc_unit = device_get_unit(dev);
337
338 snprintf(ctrlr->sc_name, sizeof(ctrlr->sc_name), "%s",
339 device_get_nameunit(dev));
340
341 mtx_init(&ctrlr->sc_mtx, device_get_nameunit(dev), NULL,
342 MTX_DEF | MTX_RECURSE);
343
344 mtx_init(&ctrlr->uic_cmd_lock, "ufshci ctrlr uic cmd lock", NULL,
345 MTX_DEF);
346
347 ver = ufshci_mmio_read_4(ctrlr, ver);
348 ctrlr->major_version = UFSHCIV(UFSHCI_VER_REG_MJR, ver);
349 ctrlr->minor_version = UFSHCIV(UFSHCI_VER_REG_MNR, ver);
350 ufshci_printf(ctrlr, "UFSHCI Version: %d.%d\n", ctrlr->major_version,
351 ctrlr->minor_version);
352
353 /* Read Device Capabilities */
354 ctrlr->cap = cap = ufshci_mmio_read_4(ctrlr, cap);
355 if (ctrlr->quirks & UFSHCI_QUIRK_BROKEN_LSDBS_MCQS_CAP) {
356 ctrlr->is_single_db_supported = true;
357 ctrlr->is_mcq_supported = true;
358 } else {
359 ctrlr->is_single_db_supported = (UFSHCIV(UFSHCI_CAP_REG_LSDBS,
360 cap) == 0);
361 ctrlr->is_mcq_supported = (UFSHCIV(UFSHCI_CAP_REG_MCQS, cap) ==
362 1);
363 }
364 if (!(ctrlr->is_single_db_supported || ctrlr->is_mcq_supported))
365 return (ENXIO);
366
367 /*
368 * The maximum transfer size supported by UFSHCI spec is 65535 * 256 KiB
369 * However, we limit the maximum transfer size to 1MiB(256 * 4KiB) for
370 * performance reason.
371 */
372 ctrlr->page_size = PAGE_SIZE;
373 ctrlr->max_xfer_size = ctrlr->page_size * UFSHCI_MAX_PRDT_ENTRY_COUNT;
374
375 timeout_period = UFSHCI_DEFAULT_TIMEOUT_PERIOD;
376 TUNABLE_INT_FETCH("hw.ufshci.timeout_period", &timeout_period);
377 timeout_period = min(timeout_period, UFSHCI_MAX_TIMEOUT_PERIOD);
378 timeout_period = max(timeout_period, UFSHCI_MIN_TIMEOUT_PERIOD);
379 ctrlr->timeout_period = timeout_period;
380
381 retry_count = UFSHCI_DEFAULT_RETRY_COUNT;
382 TUNABLE_INT_FETCH("hw.ufshci.retry_count", &retry_count);
383 ctrlr->retry_count = retry_count;
384
385 ctrlr->enable_aborts = 1;
386 if (ctrlr->quirks & UFSHCI_QUIRK_NOT_SUPPORT_ABORT_TASK)
387 ctrlr->enable_aborts = 0;
388 else
389 TUNABLE_INT_FETCH("hw.ufshci.enable_aborts",
390 &ctrlr->enable_aborts);
391
392 /* Reset the UFSHCI controller */
393 error = ufshci_ctrlr_hw_reset(ctrlr);
394 if (error)
395 return (error);
396
397 /* Read the UECPA register to clear */
398 ufshci_mmio_read_4(ctrlr, uecpa);
399
400 /* Diable Auto-hibernate */
401 ahit = 0;
402 ufshci_mmio_write_4(ctrlr, ahit, ahit);
403
404 /* Allocate and initialize UTP Task Management Request List. */
405 error = ufshci_utmr_req_queue_construct(ctrlr);
406 if (error)
407 return (error);
408
409 /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */
410 error = ufshci_utr_req_queue_construct(ctrlr);
411 if (error)
412 return (error);
413
414 /* TODO: Separate IO and Admin slot */
415
416 /*
417 * max_hw_pend_io is the number of slots in the transfer_req_queue.
418 * Reduce num_entries by one to reserve an admin slot.
419 */
420 ctrlr->max_hw_pend_io = ctrlr->transfer_req_queue.num_entries - 1;
421
422 /* Create a thread for the taskqueue. */
423 ctrlr->taskqueue = taskqueue_create("ufshci_taskq", M_WAITOK,
424 taskqueue_thread_enqueue, &ctrlr->taskqueue);
425 taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "ufshci taskq");
426
427 TASK_INIT(&ctrlr->reset_task, 0, ufshci_ctrlr_reset_task, ctrlr);
428
429 return (0);
430 }
431
432 void
ufshci_ctrlr_destruct(struct ufshci_controller * ctrlr,device_t dev)433 ufshci_ctrlr_destruct(struct ufshci_controller *ctrlr, device_t dev)
434 {
435 if (ctrlr->resource == NULL)
436 goto nores;
437
438 /* TODO: Flush In-flight IOs */
439
440 /* Release resources */
441 ufshci_utmr_req_queue_destroy(ctrlr);
442 ufshci_utr_req_queue_destroy(ctrlr);
443
444 if (ctrlr->tag)
445 bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
446
447 if (ctrlr->res)
448 bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
449 rman_get_rid(ctrlr->res), ctrlr->res);
450
451 mtx_lock(&ctrlr->sc_mtx);
452
453 ufshci_sim_detach(ctrlr);
454
455 mtx_unlock(&ctrlr->sc_mtx);
456
457 bus_release_resource(dev, SYS_RES_MEMORY, ctrlr->resource_id,
458 ctrlr->resource);
459 nores:
460 KASSERT(!mtx_owned(&ctrlr->uic_cmd_lock),
461 ("destroying uic_cmd_lock while still owned"));
462 mtx_destroy(&ctrlr->uic_cmd_lock);
463
464 KASSERT(!mtx_owned(&ctrlr->sc_mtx),
465 ("destroying sc_mtx while still owned"));
466 mtx_destroy(&ctrlr->sc_mtx);
467
468 return;
469 }
470
471 void
ufshci_ctrlr_reset(struct ufshci_controller * ctrlr)472 ufshci_ctrlr_reset(struct ufshci_controller *ctrlr)
473 {
474 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
475 }
476
477 int
ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller * ctrlr,struct ufshci_request * req)478 ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr,
479 struct ufshci_request *req)
480 {
481 return (
482 ufshci_req_queue_submit_request(&ctrlr->task_mgmt_req_queue, req));
483 }
484
485 int
ufshci_ctrlr_submit_transfer_request(struct ufshci_controller * ctrlr,struct ufshci_request * req)486 ufshci_ctrlr_submit_transfer_request(struct ufshci_controller *ctrlr,
487 struct ufshci_request *req)
488 {
489 return (
490 ufshci_req_queue_submit_request(&ctrlr->transfer_req_queue, req));
491 }
492
493 int
ufshci_ctrlr_send_nop(struct ufshci_controller * ctrlr)494 ufshci_ctrlr_send_nop(struct ufshci_controller *ctrlr)
495 {
496 struct ufshci_completion_poll_status status;
497
498 status.done = 0;
499 ufshci_ctrlr_cmd_send_nop(ctrlr, ufshci_completion_poll_cb, &status);
500 ufshci_completion_poll(&status);
501 if (status.error) {
502 ufshci_printf(ctrlr, "ufshci_ctrlr_send_nop failed!\n");
503 return (ENXIO);
504 }
505
506 return (0);
507 }
508
509 void
ufshci_ctrlr_start_config_hook(void * arg)510 ufshci_ctrlr_start_config_hook(void *arg)
511 {
512 struct ufshci_controller *ctrlr = arg;
513
514 TSENTER();
515
516 if (ufshci_utmr_req_queue_enable(ctrlr) == 0 &&
517 ufshci_utr_req_queue_enable(ctrlr) == 0)
518 ufshci_ctrlr_start(ctrlr, false);
519 else
520 ufshci_ctrlr_fail(ctrlr);
521
522 ufshci_sysctl_initialize_ctrlr(ctrlr);
523 config_intrhook_disestablish(&ctrlr->config_hook);
524
525 TSEXIT();
526 }
527
528 /*
529 * Poll all the queues enabled on the device for completion.
530 */
531 void
ufshci_ctrlr_poll(struct ufshci_controller * ctrlr)532 ufshci_ctrlr_poll(struct ufshci_controller *ctrlr)
533 {
534 uint32_t is;
535
536 is = ufshci_mmio_read_4(ctrlr, is);
537
538 /* UIC error */
539 if (is & UFSHCIM(UFSHCI_IS_REG_UE)) {
540 uint32_t uecpa, uecdl, uecn, uect, uecdme;
541
542 /* UECPA for Host UIC Error Code within PHY Adapter Layer */
543 uecpa = ufshci_mmio_read_4(ctrlr, uecpa);
544 if (uecpa & UFSHCIM(UFSHCI_UECPA_REG_ERR)) {
545 ufshci_printf(ctrlr, "UECPA error code: 0x%x\n",
546 UFSHCIV(UFSHCI_UECPA_REG_EC, uecpa));
547 }
548 /* UECDL for Host UIC Error Code within Data Link Layer */
549 uecdl = ufshci_mmio_read_4(ctrlr, uecdl);
550 if (uecdl & UFSHCIM(UFSHCI_UECDL_REG_ERR)) {
551 ufshci_printf(ctrlr, "UECDL error code: 0x%x\n",
552 UFSHCIV(UFSHCI_UECDL_REG_EC, uecdl));
553 }
554 /* UECN for Host UIC Error Code within Network Layer */
555 uecn = ufshci_mmio_read_4(ctrlr, uecn);
556 if (uecn & UFSHCIM(UFSHCI_UECN_REG_ERR)) {
557 ufshci_printf(ctrlr, "UECN error code: 0x%x\n",
558 UFSHCIV(UFSHCI_UECN_REG_EC, uecn));
559 }
560 /* UECT for Host UIC Error Code within Transport Layer */
561 uect = ufshci_mmio_read_4(ctrlr, uect);
562 if (uect & UFSHCIM(UFSHCI_UECT_REG_ERR)) {
563 ufshci_printf(ctrlr, "UECT error code: 0x%x\n",
564 UFSHCIV(UFSHCI_UECT_REG_EC, uect));
565 }
566 /* UECDME for Host UIC Error Code within DME subcomponent */
567 uecdme = ufshci_mmio_read_4(ctrlr, uecdme);
568 if (uecdme & UFSHCIM(UFSHCI_UECDME_REG_ERR)) {
569 ufshci_printf(ctrlr, "UECDME error code: 0x%x\n",
570 UFSHCIV(UFSHCI_UECDME_REG_EC, uecdme));
571 }
572 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UE));
573 }
574 /* Device Fatal Error Status */
575 if (is & UFSHCIM(UFSHCI_IS_REG_DFES)) {
576 ufshci_printf(ctrlr, "Device fatal error on ISR\n");
577 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_DFES));
578 }
579 /* UTP Error Status */
580 if (is & UFSHCIM(UFSHCI_IS_REG_UTPES)) {
581 ufshci_printf(ctrlr, "UTP error on ISR\n");
582 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTPES));
583 }
584 /* Host Controller Fatal Error Status */
585 if (is & UFSHCIM(UFSHCI_IS_REG_HCFES)) {
586 ufshci_printf(ctrlr, "Host controller fatal error on ISR\n");
587 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_HCFES));
588 }
589 /* System Bus Fatal Error Status */
590 if (is & UFSHCIM(UFSHCI_IS_REG_SBFES)) {
591 ufshci_printf(ctrlr, "System bus fatal error on ISR\n");
592 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_SBFES));
593 }
594 /* Crypto Engine Fatal Error Status */
595 if (is & UFSHCIM(UFSHCI_IS_REG_CEFES)) {
596 ufshci_printf(ctrlr, "Crypto engine fatal error on ISR\n");
597 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_CEFES));
598 }
599 /* UTP Task Management Request Completion Status */
600 if (is & UFSHCIM(UFSHCI_IS_REG_UTMRCS)) {
601 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTMRCS));
602 ufshci_req_queue_process_completions(
603 &ctrlr->task_mgmt_req_queue);
604 }
605 /* UTP Transfer Request Completion Status */
606 if (is & UFSHCIM(UFSHCI_IS_REG_UTRCS)) {
607 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTRCS));
608 ufshci_req_queue_process_completions(
609 &ctrlr->transfer_req_queue);
610 }
611 /* MCQ CQ Event Status */
612 if (is & UFSHCIM(UFSHCI_IS_REG_CQES)) {
613 /* TODO: We need to process completion Queue Pairs */
614 ufshci_printf(ctrlr, "MCQ completion not yet implemented\n");
615 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_CQES));
616 }
617 }
618
619 /*
620 * Poll the single-vector interrupt case: num_io_queues will be 1 and
621 * there's only a single vector. While we're polling, we mask further
622 * interrupts in the controller.
623 */
624 void
ufshci_ctrlr_shared_handler(void * arg)625 ufshci_ctrlr_shared_handler(void *arg)
626 {
627 struct ufshci_controller *ctrlr = arg;
628
629 ufshci_ctrlr_poll(ctrlr);
630 }
631
632 void
ufshci_reg_dump(struct ufshci_controller * ctrlr)633 ufshci_reg_dump(struct ufshci_controller *ctrlr)
634 {
635 ufshci_printf(ctrlr, "========= UFSHCI Register Dump =========\n");
636
637 UFSHCI_DUMP_REG(ctrlr, cap);
638 UFSHCI_DUMP_REG(ctrlr, mcqcap);
639 UFSHCI_DUMP_REG(ctrlr, ver);
640 UFSHCI_DUMP_REG(ctrlr, ext_cap);
641 UFSHCI_DUMP_REG(ctrlr, hcpid);
642 UFSHCI_DUMP_REG(ctrlr, hcmid);
643 UFSHCI_DUMP_REG(ctrlr, ahit);
644 UFSHCI_DUMP_REG(ctrlr, is);
645 UFSHCI_DUMP_REG(ctrlr, ie);
646 UFSHCI_DUMP_REG(ctrlr, hcsext);
647 UFSHCI_DUMP_REG(ctrlr, hcs);
648 UFSHCI_DUMP_REG(ctrlr, hce);
649 UFSHCI_DUMP_REG(ctrlr, uecpa);
650 UFSHCI_DUMP_REG(ctrlr, uecdl);
651 UFSHCI_DUMP_REG(ctrlr, uecn);
652 UFSHCI_DUMP_REG(ctrlr, uect);
653 UFSHCI_DUMP_REG(ctrlr, uecdme);
654
655 ufshci_printf(ctrlr, "========================================\n");
656 }
657
658 int
ufshci_ctrlr_suspend(struct ufshci_controller * ctrlr,enum power_stype stype)659 ufshci_ctrlr_suspend(struct ufshci_controller *ctrlr, enum power_stype stype)
660 {
661 int error;
662
663 if (!ctrlr->ufs_dev.power_mode_supported)
664 return (0);
665
666 /* TODO: Need to flush the request queue */
667
668 if (ctrlr->ufs_device_wlun_periph) {
669 ctrlr->ufs_dev.power_mode = power_map[stype].dev_pwr;
670 error = ufshci_sim_send_ssu(ctrlr, /*start*/ false,
671 power_map[stype].ssu_pc, /*immed*/ false);
672 if (error) {
673 ufshci_printf(ctrlr,
674 "Failed to send SSU in suspend handler\n");
675 return (error);
676 }
677 }
678
679 /* Change the link state */
680 error = ufshci_dev_link_state_transition(ctrlr,
681 power_map[stype].link_state);
682 if (error) {
683 ufshci_printf(ctrlr,
684 "Failed to transition link state in suspend handler\n");
685 return (error);
686 }
687
688 return (0);
689 }
690
691 int
ufshci_ctrlr_resume(struct ufshci_controller * ctrlr,enum power_stype stype)692 ufshci_ctrlr_resume(struct ufshci_controller *ctrlr, enum power_stype stype)
693 {
694 int error;
695
696 if (!ctrlr->ufs_dev.power_mode_supported)
697 return (0);
698
699 /* Change the link state */
700 error = ufshci_dev_link_state_transition(ctrlr,
701 power_map[stype].link_state);
702 if (error) {
703 ufshci_printf(ctrlr,
704 "Failed to transition link state in resume handler\n");
705 return (error);
706 }
707
708 if (ctrlr->ufs_device_wlun_periph) {
709 ctrlr->ufs_dev.power_mode = power_map[stype].dev_pwr;
710 error = ufshci_sim_send_ssu(ctrlr, /*start*/ false,
711 power_map[stype].ssu_pc, /*immed*/ false);
712 if (error) {
713 ufshci_printf(ctrlr,
714 "Failed to send SSU in resume handler\n");
715 return (error);
716 }
717 }
718
719 ufshci_dev_enable_auto_hibernate(ctrlr);
720
721 return (0);
722 }
723