1 /*- 2 * Copyright (c) 2025, Samsung Electronics Co., Ltd. 3 * Written by Jaeyoon Choi 4 * 5 * SPDX-License-Identifier: BSD-2-Clause 6 */ 7 8 #include <sys/param.h> 9 #include <sys/bus.h> 10 #include <sys/conf.h> 11 12 #include "ufshci_private.h" 13 #include "ufshci_reg.h" 14 15 static void 16 ufshci_ctrlr_fail(struct ufshci_controller *ctrlr) 17 { 18 /* 19 * The attach thread and the reset task can both fail the 20 * controller. A second queue walk would complete the same 21 * trackers again. 22 */ 23 if (atomic_swap_32(&ctrlr->is_failed, 1) != 0) 24 return; 25 26 ufshci_req_queue_fail(ctrlr, &ctrlr->task_mgmt_req_queue); 27 ufshci_req_queue_fail(ctrlr, &ctrlr->transfer_req_queue); 28 } 29 30 /* Some controllers require a reinit after switching to the max gear. */ 31 static int 32 ufshci_ctrlr_reinit_after_max_gear_switch(struct ufshci_controller *ctrlr) 33 { 34 int error; 35 36 /* Reset device */ 37 ufshci_utmr_req_queue_disable(ctrlr); 38 ufshci_utr_req_queue_disable(ctrlr); 39 40 error = ufshci_ctrlr_disable(ctrlr); 41 if (error != 0) 42 return (error); 43 44 error = ufshci_ctrlr_enable(ctrlr); 45 if (error != 0) 46 return (error); 47 48 error = ufshci_utmr_req_queue_enable(ctrlr); 49 if (error != 0) 50 return (error); 51 52 error = ufshci_utr_req_queue_enable(ctrlr); 53 if (error != 0) 54 return (error); 55 56 error = ufshci_ctrlr_send_nop(ctrlr); 57 if (error != 0) 58 return (error); 59 60 /* Reinit the target device. */ 61 error = ufshci_dev_init(ctrlr); 62 if (error != 0) 63 return (error); 64 65 /* Initialize Reference Clock */ 66 error = ufshci_dev_init_reference_clock(ctrlr); 67 if (error != 0) 68 return (error); 69 70 /* Initialize unipro */ 71 return (ufshci_dev_init_unipro(ctrlr)); 72 } 73 74 static void 75 ufshci_ctrlr_start(struct ufshci_controller *ctrlr, bool resetting) 76 { 77 TSENTER(); 78 79 /* 80 * If `resetting` is true, we are on the reset path. 81 * Re-enable request queues here because ufshci_ctrlr_reset_task() 82 * disables them during reset. 83 */ 84 if (resetting) { 85 if (ufshci_utmr_req_queue_enable(ctrlr) != 0) { 86 ufshci_ctrlr_fail(ctrlr); 87 return; 88 } 89 if (ufshci_utr_req_queue_enable(ctrlr) != 0) { 90 ufshci_ctrlr_fail(ctrlr); 91 return; 92 } 93 } 94 95 if (ufshci_ctrlr_send_nop(ctrlr) != 0) { 96 ufshci_ctrlr_fail(ctrlr); 97 return; 98 } 99 100 /* Initialize UFS target drvice */ 101 if (ufshci_dev_init(ctrlr) != 0) { 102 ufshci_ctrlr_fail(ctrlr); 103 return; 104 } 105 106 /* Initialize Reference Clock */ 107 if (ufshci_dev_init_reference_clock(ctrlr) != 0) { 108 ufshci_ctrlr_fail(ctrlr); 109 return; 110 } 111 112 /* Initialize unipro */ 113 if (ufshci_dev_init_unipro(ctrlr) != 0) { 114 ufshci_ctrlr_fail(ctrlr); 115 return; 116 } 117 118 /* 119 * Initialize UIC Power Mode 120 * QEMU UFS devices do not support unipro and power mode. 121 */ 122 if (!(ctrlr->quirks & UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE) && 123 ufshci_dev_init_uic_power_mode(ctrlr) != 0) { 124 ufshci_ctrlr_fail(ctrlr); 125 return; 126 } 127 128 ufshci_dev_init_uic_link_state(ctrlr); 129 130 if (ctrlr->quirks & UFSHCI_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { 131 uint32_t probe; 132 133 /* 134 * The reinit is only needed when the link did not survive 135 * the gear switch. A local readback still shows HS when the 136 * peer is dead. Only peer traffic proves the link works. 137 */ 138 if (ufshci_uic_send_dme_peer_get(ctrlr, PA_Granularity, 139 &probe) != 0) { 140 ufshci_printf(ctrlr, 141 "link probe failed after the gear switch, " 142 "reinitializing\n"); 143 if (ufshci_ctrlr_reinit_after_max_gear_switch( 144 ctrlr) != 0) { 145 ufshci_ctrlr_fail(ctrlr); 146 return; 147 } 148 } 149 } 150 151 /* Read Controller Descriptor (Device, Geometry) */ 152 if (ufshci_dev_get_descriptor(ctrlr) != 0) { 153 ufshci_ctrlr_fail(ctrlr); 154 return; 155 } 156 157 if (ufshci_dev_config_write_booster(ctrlr)) { 158 ufshci_ctrlr_fail(ctrlr); 159 return; 160 } 161 162 ufshci_dev_init_auto_hibernate(ctrlr); 163 164 /* TODO: Configure Write Protect */ 165 166 /* TODO: Configure Background Operations */ 167 168 /* 169 * A reset normally arrives after the SIM is attached. But if the 170 * first start attempt failed early, the reset path runs without a 171 * SIM. Attach it whenever it does not exist yet. 172 */ 173 if (ctrlr->ufshci_sim == NULL && ufshci_sim_attach(ctrlr) != 0) { 174 ufshci_ctrlr_fail(ctrlr); 175 return; 176 } 177 178 /* Initialize UFS Power Mode */ 179 if (ufshci_dev_init_ufs_power_mode(ctrlr) != 0) { 180 ufshci_ctrlr_fail(ctrlr); 181 return; 182 } 183 184 TSEXIT(); 185 } 186 187 static int 188 ufshci_ctrlr_disable_host_ctrlr(struct ufshci_controller *ctrlr) 189 { 190 int timeout = ticks + MSEC_2_TICKS(ctrlr->device_init_timeout_in_ms); 191 sbintime_t delta_t = SBT_1US; 192 uint32_t hce; 193 194 hce = ufshci_mmio_read_4(ctrlr, hce); 195 196 /* If UFS host controller is already enabled, disable it. */ 197 if (UFSHCIV(UFSHCI_HCE_REG_HCE, hce)) { 198 hce &= ~UFSHCIM(UFSHCI_HCE_REG_HCE); 199 ufshci_mmio_write_4(ctrlr, hce, hce); 200 } 201 202 /* Wait for the HCE flag to change */ 203 while (1) { 204 hce = ufshci_mmio_read_4(ctrlr, hce); 205 if (!UFSHCIV(UFSHCI_HCE_REG_HCE, hce)) 206 break; 207 if (timeout - ticks < 0) { 208 ufshci_printf(ctrlr, 209 "host controller failed to disable " 210 "within %d ms\n", 211 ctrlr->device_init_timeout_in_ms); 212 return (ENXIO); 213 } 214 215 pause_sbt("ufshci_disable_hce", delta_t, 0, C_PREL(1)); 216 delta_t = min(SBT_1MS, delta_t * 3 / 2); 217 } 218 219 return (0); 220 } 221 222 static int 223 ufshci_ctrlr_enable_host_ctrlr(struct ufshci_controller *ctrlr) 224 { 225 int timeout = ticks + MSEC_2_TICKS(ctrlr->device_init_timeout_in_ms); 226 sbintime_t delta_t = SBT_1US; 227 uint32_t hce; 228 229 hce = ufshci_mmio_read_4(ctrlr, hce); 230 231 /* Enable UFS host controller */ 232 hce |= UFSHCIM(UFSHCI_HCE_REG_HCE); 233 ufshci_mmio_write_4(ctrlr, hce, hce); 234 235 /* 236 * During the controller initialization, the value of the HCE bit is 237 * unstable, so we need to read the HCE value after some time after 238 * initialization is complete. 239 */ 240 pause_sbt("ufshci_enable_hce", ustosbt(100), 0, C_PREL(1)); 241 242 /* Wait for the HCE flag to change */ 243 while (1) { 244 hce = ufshci_mmio_read_4(ctrlr, hce); 245 if (UFSHCIV(UFSHCI_HCE_REG_HCE, hce)) 246 break; 247 if (timeout - ticks < 0) { 248 ufshci_printf(ctrlr, 249 "host controller failed to enable " 250 "within %d ms\n", 251 ctrlr->device_init_timeout_in_ms); 252 return (ENXIO); 253 } 254 255 pause_sbt("ufshci_enable_hce", delta_t, 0, C_PREL(1)); 256 delta_t = min(SBT_1MS, delta_t * 3 / 2); 257 } 258 259 return (0); 260 } 261 262 int 263 ufshci_ctrlr_disable(struct ufshci_controller *ctrlr) 264 { 265 int error; 266 267 /* Disable all interrupts */ 268 ufshci_mmio_write_4(ctrlr, ie, 0); 269 270 error = ufshci_ctrlr_disable_host_ctrlr(ctrlr); 271 return (error); 272 } 273 274 int 275 ufshci_ctrlr_enable(struct ufshci_controller *ctrlr) 276 { 277 uint32_t ie, hcs; 278 int error; 279 280 error = ufshci_ctrlr_enable_host_ctrlr(ctrlr); 281 if (error) 282 return (error); 283 284 /* Send DME_LINKSTARTUP command to start the link startup procedure */ 285 error = ufshci_uic_send_dme_link_startup(ctrlr); 286 if (error) 287 return (error); 288 289 /* 290 * The device_present(UFSHCI_HCS_REG_DP) bit becomes true if the host 291 * controller has successfully received a Link Startup UIC command 292 * response and the UFS device has found a physical link to the 293 * controller. 294 */ 295 hcs = ufshci_mmio_read_4(ctrlr, hcs); 296 if (!UFSHCIV(UFSHCI_HCS_REG_DP, hcs)) { 297 ufshci_printf(ctrlr, "UFS device not found\n"); 298 return (ENXIO); 299 } 300 301 /* Enable additional interrupts by programming the IE register. */ 302 ie = ufshci_mmio_read_4(ctrlr, ie); 303 ie |= UFSHCIM(UFSHCI_IE_REG_UTRCE); /* UTR Completion */ 304 ie |= UFSHCIM(UFSHCI_IE_REG_UEE); /* UIC Error */ 305 ie |= UFSHCIM(UFSHCI_IE_REG_UTMRCE); /* UTMR Completion */ 306 ie |= UFSHCIM(UFSHCI_IE_REG_DFEE); /* Device Fatal Error */ 307 ie |= UFSHCIM(UFSHCI_IE_REG_UTPEE); /* UTP Error */ 308 ie |= UFSHCIM(UFSHCI_IE_REG_HCFEE); /* Host Ctrlr Fatal Error */ 309 ie |= UFSHCIM(UFSHCI_IE_REG_SBFEE); /* System Bus Fatal Error */ 310 ie |= UFSHCIM(UFSHCI_IE_REG_CEFEE); /* Crypto Engine Fatal Error */ 311 ufshci_mmio_write_4(ctrlr, ie, ie); 312 313 /* TODO: Initialize interrupt Aggregation Control Register (UTRIACR) */ 314 315 return (0); 316 } 317 318 static int 319 ufshci_ctrlr_hw_reset(struct ufshci_controller *ctrlr) 320 { 321 int error; 322 323 error = ufshci_ctrlr_disable(ctrlr); 324 if (error) 325 return (error); 326 327 error = ufshci_ctrlr_enable(ctrlr); 328 return (error); 329 } 330 331 static void 332 ufshci_ctrlr_reset_task(void *arg, int pending) 333 { 334 struct ufshci_controller *ctrlr = arg; 335 int error; 336 337 /* A failed controller must not be re-enabled. */ 338 if (ctrlr->is_failed) 339 return; 340 341 /* Release resources */ 342 ufshci_utmr_req_queue_disable(ctrlr); 343 ufshci_utr_req_queue_disable(ctrlr); 344 345 error = ufshci_ctrlr_hw_reset(ctrlr); 346 if (error) 347 return (ufshci_ctrlr_fail(ctrlr)); 348 349 ufshci_ctrlr_start(ctrlr, true); 350 } 351 352 int 353 ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev) 354 { 355 uint32_t ver, cap, ahit; 356 uint32_t timeout_period, retry_count; 357 int error; 358 359 ctrlr->device_init_timeout_in_ms = UFSHCI_DEVICE_INIT_TIMEOUT_MS; 360 ctrlr->uic_cmd_timeout_in_ms = UFSHCI_UIC_CMD_TIMEOUT_MS; 361 ctrlr->dev = dev; 362 ctrlr->sc_unit = device_get_unit(dev); 363 364 snprintf(ctrlr->sc_name, sizeof(ctrlr->sc_name), "%s", 365 device_get_nameunit(dev)); 366 367 mtx_init(&ctrlr->sc_mtx, device_get_nameunit(dev), NULL, 368 MTX_DEF | MTX_RECURSE); 369 370 mtx_init(&ctrlr->uic_cmd_lock, "ufshci ctrlr uic cmd lock", NULL, 371 MTX_DEF); 372 373 ver = ufshci_mmio_read_4(ctrlr, ver); 374 ctrlr->major_version = UFSHCIV(UFSHCI_VER_REG_MJR, ver); 375 ctrlr->minor_version = UFSHCIV(UFSHCI_VER_REG_MNR, ver); 376 ufshci_printf(ctrlr, "UFSHCI Version: %d.%d\n", ctrlr->major_version, 377 ctrlr->minor_version); 378 379 /* Read Device Capabilities */ 380 ctrlr->cap = cap = ufshci_mmio_read_4(ctrlr, cap); 381 if (ctrlr->quirks & UFSHCI_QUIRK_BROKEN_LSDBS_MCQS_CAP) { 382 ctrlr->is_single_db_supported = true; 383 ctrlr->is_mcq_supported = true; 384 } else { 385 ctrlr->is_single_db_supported = (UFSHCIV(UFSHCI_CAP_REG_LSDBS, 386 cap) == 0); 387 ctrlr->is_mcq_supported = (UFSHCIV(UFSHCI_CAP_REG_MCQS, cap) == 388 1); 389 } 390 if (!(ctrlr->is_single_db_supported || ctrlr->is_mcq_supported)) 391 return (ENXIO); 392 393 /* Every device table entry must name the HS series. */ 394 if (ctrlr->hs_series == 0) { 395 ufshci_printf(ctrlr, 396 "hs_series is missing from the device table\n"); 397 return (ENXIO); 398 } 399 400 /* 401 * The maximum transfer size supported by UFSHCI spec is 65535 * 256 KiB 402 * However, we limit the maximum transfer size to 1MiB(256 * 4KiB) for 403 * performance reason. 404 */ 405 ctrlr->page_size = PAGE_SIZE; 406 ctrlr->max_xfer_size = ctrlr->page_size * UFSHCI_MAX_PRDT_ENTRY_COUNT; 407 408 timeout_period = UFSHCI_DEFAULT_TIMEOUT_PERIOD; 409 TUNABLE_INT_FETCH("hw.ufshci.timeout_period", &timeout_period); 410 timeout_period = min(timeout_period, UFSHCI_MAX_TIMEOUT_PERIOD); 411 timeout_period = max(timeout_period, UFSHCI_MIN_TIMEOUT_PERIOD); 412 ctrlr->timeout_period = timeout_period; 413 414 retry_count = UFSHCI_DEFAULT_RETRY_COUNT; 415 TUNABLE_INT_FETCH("hw.ufshci.retry_count", &retry_count); 416 ctrlr->retry_count = retry_count; 417 418 ctrlr->enable_aborts = 1; 419 if (ctrlr->quirks & UFSHCI_QUIRK_NOT_SUPPORT_ABORT_TASK) 420 ctrlr->enable_aborts = 0; 421 else 422 TUNABLE_INT_FETCH("hw.ufshci.enable_aborts", 423 &ctrlr->enable_aborts); 424 425 /* Reset the UFSHCI controller */ 426 error = ufshci_ctrlr_hw_reset(ctrlr); 427 if (error) 428 return (error); 429 430 /* Read the UECPA register to clear */ 431 ufshci_mmio_read_4(ctrlr, uecpa); 432 433 /* Disable Auto-hibernate */ 434 ahit = 0; 435 ufshci_mmio_write_4(ctrlr, ahit, ahit); 436 437 /* Allocate and initialize UTP Task Management Request List. */ 438 error = ufshci_utmr_req_queue_construct(ctrlr); 439 if (error) 440 return (error); 441 442 /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */ 443 error = ufshci_utr_req_queue_construct(ctrlr); 444 if (error) 445 return (error); 446 447 /* TODO: Separate IO and Admin slot */ 448 449 /* 450 * max_hw_pend_io is the number of slots in the transfer_req_queue. 451 * Reduce num_entries by one to reserve an admin slot. 452 */ 453 ctrlr->max_hw_pend_io = ctrlr->transfer_req_queue.num_entries - 1; 454 455 /* Create a thread for the taskqueue. */ 456 ctrlr->taskqueue = taskqueue_create("ufshci_taskq", M_WAITOK, 457 taskqueue_thread_enqueue, &ctrlr->taskqueue); 458 taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "ufshci taskq"); 459 460 TASK_INIT(&ctrlr->reset_task, 0, ufshci_ctrlr_reset_task, ctrlr); 461 462 error = ufshci_ioctl_construct(ctrlr, dev); 463 if (error) 464 return (error); 465 466 return (0); 467 } 468 469 void 470 ufshci_ctrlr_destruct(struct ufshci_controller *ctrlr, device_t dev) 471 { 472 ufshci_ioctl_destruct(ctrlr); 473 474 if (ctrlr->resource == NULL) 475 goto nores; 476 477 /* TODO: Flush In-flight IOs */ 478 479 /* Release resources */ 480 ufshci_utmr_req_queue_destroy(ctrlr); 481 ufshci_utr_req_queue_destroy(ctrlr); 482 483 if (ctrlr->tag) 484 bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag); 485 486 if (ctrlr->res) 487 bus_release_resource(ctrlr->dev, SYS_RES_IRQ, 488 rman_get_rid(ctrlr->res), ctrlr->res); 489 490 /* 491 * The interrupt and the timers are gone, so nothing enqueues new 492 * tasks. Free the taskqueue before the SIM teardown below. 493 */ 494 if (ctrlr->taskqueue != NULL) { 495 taskqueue_free(ctrlr->taskqueue); 496 ctrlr->taskqueue = NULL; 497 } 498 499 ufshci_sim_release_wlun_periph(ctrlr); 500 501 mtx_lock(&ctrlr->sc_mtx); 502 503 ufshci_sim_detach(ctrlr); 504 505 mtx_unlock(&ctrlr->sc_mtx); 506 507 bus_release_resource(dev, SYS_RES_MEMORY, ctrlr->resource_id, 508 ctrlr->resource); 509 nores: 510 KASSERT(!mtx_owned(&ctrlr->uic_cmd_lock), 511 ("destroying uic_cmd_lock while still owned")); 512 mtx_destroy(&ctrlr->uic_cmd_lock); 513 514 KASSERT(!mtx_owned(&ctrlr->sc_mtx), 515 ("destroying sc_mtx while still owned")); 516 mtx_destroy(&ctrlr->sc_mtx); 517 518 return; 519 } 520 521 void 522 ufshci_ctrlr_reset(struct ufshci_controller *ctrlr) 523 { 524 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task); 525 } 526 527 int 528 ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr, 529 struct ufshci_request *req) 530 { 531 return ( 532 ufshci_req_queue_submit_request(&ctrlr->task_mgmt_req_queue, req)); 533 } 534 535 int 536 ufshci_ctrlr_submit_transfer_request(struct ufshci_controller *ctrlr, 537 struct ufshci_request *req) 538 { 539 return ( 540 ufshci_req_queue_submit_request(&ctrlr->transfer_req_queue, req)); 541 } 542 543 int 544 ufshci_ctrlr_send_nop(struct ufshci_controller *ctrlr) 545 { 546 struct ufshci_completion_poll_status status; 547 int error; 548 549 status.done = 0; 550 error = ufshci_ctrlr_cmd_send_nop(ctrlr, ufshci_completion_poll_cb, 551 &status); 552 if (error) 553 return (error); 554 555 ufshci_completion_poll(&status); 556 if (status.error) { 557 ufshci_printf(ctrlr, "ufshci_ctrlr_send_nop failed!\n"); 558 return (ENXIO); 559 } 560 561 return (0); 562 } 563 564 void 565 ufshci_ctrlr_start_config_hook(void *arg) 566 { 567 struct ufshci_controller *ctrlr = arg; 568 569 TSENTER(); 570 571 if (ufshci_utmr_req_queue_enable(ctrlr) == 0 && 572 ufshci_utr_req_queue_enable(ctrlr) == 0) 573 ufshci_ctrlr_start(ctrlr, false); 574 else 575 ufshci_ctrlr_fail(ctrlr); 576 577 ufshci_sysctl_initialize_ctrlr(ctrlr); 578 config_intrhook_disestablish(&ctrlr->config_hook); 579 580 TSEXIT(); 581 } 582 583 /* 584 * Poll all the queues enabled on the device for completion. 585 */ 586 void 587 ufshci_ctrlr_poll(struct ufshci_controller *ctrlr) 588 { 589 uint32_t is; 590 591 is = ufshci_mmio_read_4(ctrlr, is); 592 593 /* UIC error */ 594 if (is & UFSHCIM(UFSHCI_IS_REG_UE)) { 595 uint32_t uecpa, uecdl, uecn, uect, uecdme; 596 597 /* UECPA for Host UIC Error Code within PHY Adapter Layer */ 598 uecpa = ufshci_mmio_read_4(ctrlr, uecpa); 599 if (uecpa & UFSHCIM(UFSHCI_UECPA_REG_ERR)) { 600 ufshci_printf(ctrlr, "UECPA error code: 0x%x\n", 601 UFSHCIV(UFSHCI_UECPA_REG_EC, uecpa)); 602 } 603 /* UECDL for Host UIC Error Code within Data Link Layer */ 604 uecdl = ufshci_mmio_read_4(ctrlr, uecdl); 605 if (uecdl & UFSHCIM(UFSHCI_UECDL_REG_ERR)) { 606 ufshci_printf(ctrlr, "UECDL error code: 0x%x\n", 607 UFSHCIV(UFSHCI_UECDL_REG_EC, uecdl)); 608 } 609 /* UECN for Host UIC Error Code within Network Layer */ 610 uecn = ufshci_mmio_read_4(ctrlr, uecn); 611 if (uecn & UFSHCIM(UFSHCI_UECN_REG_ERR)) { 612 ufshci_printf(ctrlr, "UECN error code: 0x%x\n", 613 UFSHCIV(UFSHCI_UECN_REG_EC, uecn)); 614 } 615 /* UECT for Host UIC Error Code within Transport Layer */ 616 uect = ufshci_mmio_read_4(ctrlr, uect); 617 if (uect & UFSHCIM(UFSHCI_UECT_REG_ERR)) { 618 ufshci_printf(ctrlr, "UECT error code: 0x%x\n", 619 UFSHCIV(UFSHCI_UECT_REG_EC, uect)); 620 } 621 /* UECDME for Host UIC Error Code within DME subcomponent */ 622 uecdme = ufshci_mmio_read_4(ctrlr, uecdme); 623 if (uecdme & UFSHCIM(UFSHCI_UECDME_REG_ERR)) { 624 ufshci_printf(ctrlr, "UECDME error code: 0x%x\n", 625 UFSHCIV(UFSHCI_UECDME_REG_EC, uecdme)); 626 } 627 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UE)); 628 } 629 /* Device Fatal Error Status */ 630 if (is & UFSHCIM(UFSHCI_IS_REG_DFES)) { 631 ufshci_printf(ctrlr, "Device fatal error on ISR\n"); 632 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_DFES)); 633 } 634 /* UTP Error Status */ 635 if (is & UFSHCIM(UFSHCI_IS_REG_UTPES)) { 636 ufshci_printf(ctrlr, "UTP error on ISR\n"); 637 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTPES)); 638 } 639 /* Host Controller Fatal Error Status */ 640 if (is & UFSHCIM(UFSHCI_IS_REG_HCFES)) { 641 ufshci_printf(ctrlr, "Host controller fatal error on ISR\n"); 642 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_HCFES)); 643 } 644 /* System Bus Fatal Error Status */ 645 if (is & UFSHCIM(UFSHCI_IS_REG_SBFES)) { 646 ufshci_printf(ctrlr, "System bus fatal error on ISR\n"); 647 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_SBFES)); 648 } 649 /* Crypto Engine Fatal Error Status */ 650 if (is & UFSHCIM(UFSHCI_IS_REG_CEFES)) { 651 ufshci_printf(ctrlr, "Crypto engine fatal error on ISR\n"); 652 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_CEFES)); 653 } 654 /* UTP Task Management Request Completion Status */ 655 if (is & UFSHCIM(UFSHCI_IS_REG_UTMRCS)) { 656 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTMRCS)); 657 ufshci_req_queue_process_completions( 658 &ctrlr->task_mgmt_req_queue); 659 } 660 /* UTP Transfer Request Completion Status */ 661 if (is & UFSHCIM(UFSHCI_IS_REG_UTRCS)) { 662 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTRCS)); 663 ufshci_req_queue_process_completions( 664 &ctrlr->transfer_req_queue); 665 } 666 /* MCQ CQ Event Status */ 667 if (is & UFSHCIM(UFSHCI_IS_REG_CQES)) { 668 /* TODO: We need to process completion Queue Pairs */ 669 ufshci_printf(ctrlr, "MCQ completion not yet implemented\n"); 670 ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_CQES)); 671 } 672 } 673 674 /* 675 * Poll the single-vector interrupt case: num_io_queues will be 1 and 676 * there's only a single vector. While we're polling, we mask further 677 * interrupts in the controller. 678 */ 679 void 680 ufshci_ctrlr_shared_handler(void *arg) 681 { 682 struct ufshci_controller *ctrlr = arg; 683 684 ufshci_ctrlr_poll(ctrlr); 685 } 686 687 void 688 ufshci_reg_dump(struct ufshci_controller *ctrlr) 689 { 690 ufshci_printf(ctrlr, "========= UFSHCI Register Dump =========\n"); 691 692 UFSHCI_DUMP_REG(ctrlr, cap); 693 UFSHCI_DUMP_REG(ctrlr, mcqcap); 694 UFSHCI_DUMP_REG(ctrlr, ver); 695 UFSHCI_DUMP_REG(ctrlr, ext_cap); 696 UFSHCI_DUMP_REG(ctrlr, hcpid); 697 UFSHCI_DUMP_REG(ctrlr, hcmid); 698 UFSHCI_DUMP_REG(ctrlr, ahit); 699 UFSHCI_DUMP_REG(ctrlr, is); 700 UFSHCI_DUMP_REG(ctrlr, ie); 701 UFSHCI_DUMP_REG(ctrlr, hcsext); 702 UFSHCI_DUMP_REG(ctrlr, hcs); 703 UFSHCI_DUMP_REG(ctrlr, hce); 704 UFSHCI_DUMP_REG(ctrlr, uecpa); 705 UFSHCI_DUMP_REG(ctrlr, uecdl); 706 UFSHCI_DUMP_REG(ctrlr, uecn); 707 UFSHCI_DUMP_REG(ctrlr, uect); 708 UFSHCI_DUMP_REG(ctrlr, uecdme); 709 710 ufshci_printf(ctrlr, "========================================\n"); 711 } 712 713 int 714 ufshci_ctrlr_suspend(struct ufshci_controller *ctrlr, enum power_stype stype) 715 { 716 int error; 717 718 if (!ctrlr->ufs_dev.power_mode_supported) 719 return (0); 720 721 /* TODO: Need to flush the request queue */ 722 723 if (ctrlr->ufs_device_wlun_periph) { 724 ctrlr->ufs_dev.power_mode = power_map[stype].dev_pwr; 725 error = ufshci_sim_send_ssu(ctrlr, /*start*/ false, 726 power_map[stype].ssu_pc, /*immed*/ false); 727 if (error) { 728 ufshci_printf(ctrlr, 729 "Failed to send SSU in suspend handler\n"); 730 return (error); 731 } 732 } 733 734 /* Change the link state */ 735 error = ufshci_dev_link_state_transition(ctrlr, 736 power_map[stype].link_state); 737 if (error) { 738 ufshci_printf(ctrlr, 739 "Failed to transition link state in suspend handler\n"); 740 return (error); 741 } 742 743 return (0); 744 } 745 746 int 747 ufshci_ctrlr_resume(struct ufshci_controller *ctrlr, enum power_stype stype) 748 { 749 int error; 750 751 if (!ctrlr->ufs_dev.power_mode_supported) 752 return (0); 753 754 /* Change the link state */ 755 error = ufshci_dev_link_state_transition(ctrlr, 756 power_map[stype].link_state); 757 if (error) { 758 ufshci_printf(ctrlr, 759 "Failed to transition link state in resume handler\n"); 760 return (error); 761 } 762 763 if (ctrlr->ufs_device_wlun_periph) { 764 ctrlr->ufs_dev.power_mode = power_map[stype].dev_pwr; 765 error = ufshci_sim_send_ssu(ctrlr, /*start*/ false, 766 power_map[stype].ssu_pc, /*immed*/ false); 767 if (error) { 768 ufshci_printf(ctrlr, 769 "Failed to send SSU in resume handler\n"); 770 return (error); 771 } 772 } 773 774 ufshci_dev_enable_auto_hibernate(ctrlr); 775 776 return (0); 777 } 778