1 /*- 2 * Copyright (c) 2025, Samsung Electronics Co., Ltd. 3 * Written by Jaeyoon Choi 4 * 5 * SPDX-License-Identifier: BSD-2-Clause 6 */ 7 8 #ifndef __UFSHCI_PRIVATE_H__ 9 #define __UFSHCI_PRIVATE_H__ 10 11 #ifdef _KERNEL 12 #include <sys/types.h> 13 #else /* !_KERNEL */ 14 #include <stdbool.h> 15 #include <stdint.h> 16 #endif /* _KERNEL */ 17 18 #include <sys/param.h> 19 #include <sys/systm.h> 20 #include <sys/bio.h> 21 #include <sys/bus.h> 22 #include <sys/counter.h> 23 #include <sys/kernel.h> 24 #include <sys/lock.h> 25 #include <sys/malloc.h> 26 #include <sys/memdesc.h> 27 #include <sys/module.h> 28 #include <sys/mutex.h> 29 #include <sys/rman.h> 30 #include <sys/taskqueue.h> 31 32 #include <machine/bus.h> 33 34 #include "ufshci.h" 35 36 MALLOC_DECLARE(M_UFSHCI); 37 38 #define UFSHCI_DEVICE_INIT_TIMEOUT_MS (2000) /* in milliseconds */ 39 #define UFSHCI_UIC_CMD_TIMEOUT_MS (500) /* in milliseconds */ 40 #define UFSHCI_DEFAULT_TIMEOUT_PERIOD (10) /* in seconds */ 41 #define UFSHCI_MIN_TIMEOUT_PERIOD (5) /* in seconds */ 42 #define UFSHCI_MAX_TIMEOUT_PERIOD (120) /* in seconds */ 43 44 #define UFSHCI_DEFAULT_RETRY_COUNT (4) 45 46 #define UFSHCI_UTR_ENTRIES (32) 47 #define UFSHCI_UTRM_ENTRIES (8) 48 49 #define UFSHCI_SECTOR_SIZE (512) 50 51 struct ufshci_controller; 52 53 struct ufshci_completion_poll_status { 54 struct ufshci_completion cpl; 55 int done; 56 bool error; 57 }; 58 59 struct ufshci_request { 60 struct ufshci_upiu request_upiu; 61 size_t request_size; 62 size_t response_size; 63 64 struct memdesc payload; 65 enum ufshci_data_direction data_direction; 66 ufshci_cb_fn_t cb_fn; 67 void *cb_arg; 68 bool is_admin; 69 int32_t retries; 70 bool payload_valid; 71 bool spare[2]; /* Future use */ 72 STAILQ_ENTRY(ufshci_request) stailq; 73 }; 74 75 enum ufshci_slot_state { 76 UFSHCI_SLOT_STATE_FREE = 0x0, 77 UFSHCI_SLOT_STATE_RESERVED = 0x1, 78 UFSHCI_SLOT_STATE_SCHEDULED = 0x2, 79 UFSHCI_SLOT_STATE_TIMEOUT = 0x3, 80 UFSHCI_SLOT_STATE_NEED_ERROR_HANDLING = 0x4, 81 }; 82 83 struct ufshci_tracker { 84 TAILQ_ENTRY(ufshci_tracker) tailq; 85 struct ufshci_request *req; 86 struct ufshci_req_queue *req_queue; 87 struct ufshci_hw_queue *hwq; 88 uint8_t slot_num; 89 enum ufshci_slot_state slot_state; 90 size_t response_size; 91 sbintime_t deadline; 92 93 bus_dmamap_t payload_dma_map; 94 uint64_t payload_addr; 95 96 struct ufshci_utp_cmd_desc *ucd; 97 bus_addr_t ucd_bus_addr; 98 99 uint16_t prdt_off; 100 uint16_t prdt_entry_cnt; 101 }; 102 103 enum ufshci_queue_mode { 104 UFSHCI_Q_MODE_SDB = 0x00, /* Single Doorbell Mode*/ 105 UFSHCI_Q_MODE_MCQ = 0x01, /* Multi-Circular Queue Mode*/ 106 }; 107 108 /* 109 * UFS uses slot-based Single Doorbell (SDB) mode for request submission by 110 * default and additionally supports Multi-Circular Queue (MCQ) in UFS 4.0. To 111 * minimize duplicated code between SDB and MCQ, mode dependent operations are 112 * extracted into ufshci_qops. 113 */ 114 struct ufshci_qops { 115 int (*construct)(struct ufshci_controller *ctrlr, 116 struct ufshci_req_queue *req_queue, uint32_t num_entries, 117 bool is_task_mgmt); 118 void (*destroy)(struct ufshci_controller *ctrlr, 119 struct ufshci_req_queue *req_queue); 120 struct ufshci_hw_queue *(*get_hw_queue)( 121 struct ufshci_req_queue *req_queue); 122 int (*enable)(struct ufshci_controller *ctrlr, 123 struct ufshci_req_queue *req_queue); 124 void (*disable)(struct ufshci_controller *ctrlr, 125 struct ufshci_req_queue *req_queue); 126 int (*reserve_slot)(struct ufshci_req_queue *req_queue, 127 struct ufshci_tracker **tr); 128 int (*reserve_admin_slot)(struct ufshci_req_queue *req_queue, 129 struct ufshci_tracker **tr); 130 void (*ring_doorbell)(struct ufshci_controller *ctrlr, 131 struct ufshci_tracker *tr); 132 bool (*is_doorbell_cleared)(struct ufshci_controller *ctrlr, 133 uint8_t slot); 134 void (*clear_cpl_ntf)(struct ufshci_controller *ctrlr, 135 struct ufshci_tracker *tr); 136 bool (*process_cpl)(struct ufshci_req_queue *req_queue); 137 int (*get_inflight_io)(struct ufshci_controller *ctrlr); 138 }; 139 140 #define UFSHCI_SDB_Q 0 /* Queue number for a single doorbell queue */ 141 142 enum ufshci_recovery { 143 RECOVERY_NONE = 0, /* Normal operations */ 144 RECOVERY_WAITING, /* waiting for the reset to complete */ 145 }; 146 147 /* 148 * Generic queue container used by both SDB (fixed 32-slot bitmap) and MCQ 149 * (ring buffer) modes. Fields are shared; some such as sq_head, sq_tail and 150 * cq_head are not used in SDB but used in MCQ. 151 */ 152 struct ufshci_hw_queue { 153 struct ufshci_controller *ctrlr; 154 struct ufshci_req_queue *req_queue; 155 uint32_t id; 156 int domain; 157 int cpu; 158 159 struct callout timer; /* recovery lock */ 160 bool timer_armed; /* recovery lock */ 161 enum ufshci_recovery recovery_state; /* recovery lock */ 162 163 union { 164 struct ufshci_utp_xfer_req_desc *utrd; 165 struct ufshci_utp_task_mgmt_req_desc *utmrd; 166 }; 167 168 bus_dma_tag_t dma_tag_queue; 169 bus_dmamap_t queuemem_map; 170 bus_addr_t req_queue_addr; 171 172 bus_addr_t *ucd_bus_addr; 173 174 uint32_t num_entries; 175 uint32_t num_trackers; 176 177 TAILQ_HEAD(, ufshci_tracker) free_tr; 178 TAILQ_HEAD(, ufshci_tracker) outstanding_tr; 179 180 /* 181 * A Request List using the single doorbell method uses a dedicated 182 * ufshci_tracker, one per slot. 183 */ 184 struct ufshci_tracker **act_tr; 185 186 uint32_t sq_head; /* MCQ mode */ 187 uint32_t sq_tail; /* MCQ mode */ 188 uint32_t cq_head; /* MCQ mode */ 189 190 uint32_t phase; 191 int64_t num_cmds; 192 int64_t num_intr_handler_calls; 193 int64_t num_retries; 194 int64_t num_failures; 195 196 /* 197 * Each lock may be acquired independently. 198 * When both are required, acquire them in this order to avoid 199 * deadlocks. (recovery_lock -> qlock) 200 */ 201 struct mtx_padalign qlock; 202 struct mtx_padalign recovery_lock; 203 }; 204 205 struct ufshci_req_queue { 206 struct ufshci_controller *ctrlr; 207 int domain; 208 209 /* 210 * queue_mode: active transfer scheme 211 * UFSHCI_Q_MODE_SDB – legacy single‑doorbell list 212 * UFSHCI_Q_MODE_MCQ – modern multi‑circular queue (UFSHCI 4.0+) 213 */ 214 enum ufshci_queue_mode queue_mode; 215 216 uint8_t num_q; 217 struct ufshci_hw_queue *hwq; 218 219 struct ufshci_qops qops; 220 221 bool is_task_mgmt; 222 uint32_t num_entries; 223 uint32_t num_trackers; 224 225 /* Shared DMA resource */ 226 struct ufshci_utp_cmd_desc *ucd; 227 228 bus_dma_tag_t dma_tag_ucd; 229 bus_dma_tag_t dma_tag_payload; 230 231 bus_dmamap_t ucdmem_map; 232 }; 233 234 struct ufshci_device { 235 uint32_t max_lun_count; 236 237 struct ufshci_device_descriptor dev_desc; 238 struct ufshci_geometry_descriptor geo_desc; 239 240 uint32_t unipro_version; 241 242 /* WriteBooster */ 243 bool is_wb_enabled; 244 bool is_wb_flush_enabled; 245 uint32_t wb_buffer_type; 246 uint32_t wb_buffer_size_mb; 247 uint32_t wb_user_space_config_option; 248 uint8_t wb_dedicated_lu; 249 uint32_t write_booster_flush_threshold; 250 }; 251 252 /* 253 * One of these per allocated device. 254 */ 255 struct ufshci_controller { 256 device_t dev; 257 258 uint32_t quirks; 259 #define UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE \ 260 1 /* QEMU does not support UIC POWER MODE */ 261 #define UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE \ 262 2 /* Need an additional 200 ms of PA_TActivate */ 263 #define UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE \ 264 4 /* Need to wait 1250us after power mode change */ 265 #define UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY \ 266 8 /* Need to change the number of lanes before changing HS-GEAR. */ 267 #define UFSHCI_QUIRK_NOT_SUPPORT_ABORT_TASK \ 268 16 /* QEMU does not support Task Management Request */ 269 270 uint32_t ref_clk; 271 272 struct cam_sim *ufshci_sim; 273 struct cam_path *ufshci_path; 274 275 struct mtx sc_mtx; 276 uint32_t sc_unit; 277 uint8_t sc_name[16]; 278 279 struct ufshci_device ufs_dev; 280 281 bus_space_tag_t bus_tag; 282 bus_space_handle_t bus_handle; 283 int resource_id; 284 struct resource *resource; 285 286 /* Currently, there is no UFSHCI that supports MSI, MSI-X. */ 287 int msi_count; 288 289 /* Fields for tracking progress during controller initialization. */ 290 struct intr_config_hook config_hook; 291 292 struct task reset_task; 293 struct taskqueue *taskqueue; 294 295 /* For shared legacy interrupt. */ 296 int rid; 297 struct resource *res; 298 void *tag; 299 300 uint32_t major_version; 301 uint32_t minor_version; 302 303 uint32_t enable_aborts; 304 305 uint32_t num_io_queues; 306 uint32_t max_hw_pend_io; 307 308 /* Maximum logical unit number */ 309 uint32_t max_lun_count; 310 311 /* Maximum i/o size in bytes */ 312 uint32_t max_xfer_size; 313 314 /* Controller capacity */ 315 uint32_t cap; 316 317 /* Page size and log2(page_size) - 12 that we're currently using */ 318 uint32_t page_size; 319 320 /* Timeout value on device initialization */ 321 uint32_t device_init_timeout_in_ms; 322 323 /* Timeout value on UIC command */ 324 uint32_t uic_cmd_timeout_in_ms; 325 326 /* UTMR/UTR queue timeout period in seconds */ 327 uint32_t timeout_period; 328 329 /* UTMR/UTR queue retry count */ 330 uint32_t retry_count; 331 332 /* UFS Host Controller Interface Registers */ 333 struct ufshci_registers *regs; 334 335 /* UFS Transport Protocol Layer (UTP) */ 336 struct ufshci_req_queue task_mgmt_req_queue; 337 struct ufshci_req_queue transfer_req_queue; 338 bool is_single_db_supported; /* 0 = supported */ 339 bool is_mcq_supported; /* 1 = supported */ 340 341 /* UFS Interconnect Layer (UIC) */ 342 struct mtx uic_cmd_lock; 343 uint32_t unipro_version; 344 uint8_t hs_gear; 345 uint32_t tx_lanes; 346 uint32_t rx_lanes; 347 uint32_t max_rx_hs_gear; 348 uint32_t max_tx_lanes; 349 uint32_t max_rx_lanes; 350 351 bool is_failed; 352 }; 353 354 #define ufshci_mmio_offsetof(reg) offsetof(struct ufshci_registers, reg) 355 356 #define ufshci_mmio_read_4(sc, reg) \ 357 bus_space_read_4((sc)->bus_tag, (sc)->bus_handle, \ 358 ufshci_mmio_offsetof(reg)) 359 360 #define ufshci_mmio_write_4(sc, reg, val) \ 361 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \ 362 ufshci_mmio_offsetof(reg), val) 363 364 #define ufshci_printf(ctrlr, fmt, args...) \ 365 device_printf(ctrlr->dev, fmt, ##args) 366 367 /* UFSHCI */ 368 void ufshci_completion_poll_cb(void *arg, const struct ufshci_completion *cpl, 369 bool error); 370 371 /* SIM */ 372 int ufshci_sim_attach(struct ufshci_controller *ctrlr); 373 void ufshci_sim_detach(struct ufshci_controller *ctrlr); 374 375 /* Controller */ 376 int ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev); 377 void ufshci_ctrlr_destruct(struct ufshci_controller *ctrlr, device_t dev); 378 void ufshci_ctrlr_reset(struct ufshci_controller *ctrlr); 379 /* ctrlr defined as void * to allow use with config_intrhook. */ 380 void ufshci_ctrlr_start_config_hook(void *arg); 381 void ufshci_ctrlr_poll(struct ufshci_controller *ctrlr); 382 383 int ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr, 384 struct ufshci_request *req); 385 int ufshci_ctrlr_submit_admin_request(struct ufshci_controller *ctrlr, 386 struct ufshci_request *req); 387 int ufshci_ctrlr_submit_io_request(struct ufshci_controller *ctrlr, 388 struct ufshci_request *req); 389 int ufshci_ctrlr_send_nop(struct ufshci_controller *ctrlr); 390 391 void ufshci_reg_dump(struct ufshci_controller *ctrlr); 392 393 /* Device */ 394 int ufshci_dev_init(struct ufshci_controller *ctrlr); 395 int ufshci_dev_reset(struct ufshci_controller *ctrlr); 396 int ufshci_dev_init_reference_clock(struct ufshci_controller *ctrlr); 397 int ufshci_dev_init_unipro(struct ufshci_controller *ctrlr); 398 int ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr); 399 int ufshci_dev_init_ufs_power_mode(struct ufshci_controller *ctrlr); 400 int ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr); 401 int ufshci_dev_config_write_booster(struct ufshci_controller *ctrlr); 402 403 /* Controller Command */ 404 void ufshci_ctrlr_cmd_send_task_mgmt_request(struct ufshci_controller *ctrlr, 405 ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t function, uint8_t lun, 406 uint8_t task_tag, uint8_t iid); 407 void ufshci_ctrlr_cmd_send_nop(struct ufshci_controller *ctrlr, 408 ufshci_cb_fn_t cb_fn, void *cb_arg); 409 void ufshci_ctrlr_cmd_send_query_request(struct ufshci_controller *ctrlr, 410 ufshci_cb_fn_t cb_fn, void *cb_arg, struct ufshci_query_param param); 411 void ufshci_ctrlr_cmd_send_scsi_command(struct ufshci_controller *ctrlr, 412 ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t *cmd_ptr, uint8_t cmd_len, 413 uint32_t data_len, uint8_t lun, bool is_write); 414 415 /* Request Queue */ 416 bool ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue); 417 int ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr); 418 int ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr); 419 void ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr); 420 void ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr); 421 void ufshci_utmr_req_queue_disable(struct ufshci_controller *ctrlr); 422 int ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr); 423 void ufshci_utr_req_queue_disable(struct ufshci_controller *ctrlr); 424 int ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr); 425 void ufshci_req_queue_fail(struct ufshci_controller *ctrlr, 426 struct ufshci_hw_queue *hwq); 427 int ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue, 428 struct ufshci_request *req, bool is_admin); 429 void ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr); 430 431 /* Request Single Doorbell Queue */ 432 int ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, 433 struct ufshci_req_queue *req_queue, uint32_t num_entries, 434 bool is_task_mgmt); 435 void ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr, 436 struct ufshci_req_queue *req_queue); 437 struct ufshci_hw_queue *ufshci_req_sdb_get_hw_queue( 438 struct ufshci_req_queue *req_queue); 439 void ufshci_req_sdb_disable(struct ufshci_controller *ctrlr, 440 struct ufshci_req_queue *req_queue); 441 int ufshci_req_sdb_enable(struct ufshci_controller *ctrlr, 442 struct ufshci_req_queue *req_queue); 443 int ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue, 444 struct ufshci_tracker **tr); 445 void ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr, 446 struct ufshci_tracker *tr); 447 void ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr, 448 struct ufshci_tracker *tr); 449 bool ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr, 450 uint8_t slot); 451 bool ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr, 452 uint8_t slot); 453 void ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr, 454 struct ufshci_tracker *tr); 455 void ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller *ctrlr, 456 struct ufshci_tracker *tr); 457 bool ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue); 458 int ufshci_req_sdb_get_inflight_io(struct ufshci_controller *ctrlr); 459 460 /* UIC Command */ 461 int ufshci_uic_power_mode_ready(struct ufshci_controller *ctrlr); 462 int ufshci_uic_cmd_ready(struct ufshci_controller *ctrlr); 463 int ufshci_uic_send_dme_link_startup(struct ufshci_controller *ctrlr); 464 int ufshci_uic_send_dme_get(struct ufshci_controller *ctrlr, uint16_t attribute, 465 uint32_t *return_value); 466 int ufshci_uic_send_dme_set(struct ufshci_controller *ctrlr, uint16_t attribute, 467 uint32_t value); 468 int ufshci_uic_send_dme_peer_get(struct ufshci_controller *ctrlr, 469 uint16_t attribute, uint32_t *return_value); 470 int ufshci_uic_send_dme_peer_set(struct ufshci_controller *ctrlr, 471 uint16_t attribute, uint32_t value); 472 int ufshci_uic_send_dme_endpoint_reset(struct ufshci_controller *ctrlr); 473 474 /* SYSCTL */ 475 void ufshci_sysctl_initialize_ctrlr(struct ufshci_controller *ctrlr); 476 477 int ufshci_attach(device_t dev); 478 int ufshci_detach(device_t dev); 479 480 /* 481 * Wait for a command to complete using the ufshci_completion_poll_cb. Used in 482 * limited contexts where the caller knows it's OK to block briefly while the 483 * command runs. The ISR will run the callback which will set status->done to 484 * true, usually within microseconds. If not, then after one second timeout 485 * handler should reset the controller and abort all outstanding requests 486 * including this polled one. If still not after ten seconds, then something is 487 * wrong with the driver, and panic is the only way to recover. 488 * 489 * Most commands using this interface aren't actual I/O to the drive's media so 490 * complete within a few microseconds. Adaptively spin for one tick to catch the 491 * vast majority of these without waiting for a tick plus scheduling delays. 492 * Since these are on startup, this drastically reduces startup time. 493 */ 494 static __inline void 495 ufshci_completion_poll(struct ufshci_completion_poll_status *status) 496 { 497 int timeout = ticks + 10 * hz; 498 sbintime_t delta_t = SBT_1US; 499 500 while (!atomic_load_acq_int(&status->done)) { 501 if (timeout - ticks < 0) 502 panic( 503 "UFSHCI polled command failed to complete within 10s."); 504 pause_sbt("ufshci_cpl", delta_t, 0, C_PREL(1)); 505 delta_t = min(SBT_1MS, delta_t * 3 / 2); 506 } 507 } 508 509 static __inline void 510 ufshci_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) 511 { 512 uint64_t *bus_addr = (uint64_t *)arg; 513 514 KASSERT(nseg == 1, ("number of segments (%d) is not 1", nseg)); 515 if (error != 0) 516 printf("ufshci_single_map err %d\n", error); 517 *bus_addr = seg[0].ds_addr; 518 } 519 520 static __inline struct ufshci_request * 521 _ufshci_allocate_request(const int how, ufshci_cb_fn_t cb_fn, void *cb_arg) 522 { 523 struct ufshci_request *req; 524 525 KASSERT(how == M_WAITOK || how == M_NOWAIT, 526 ("ufshci_allocate_request: invalid how %d", how)); 527 528 req = malloc(sizeof(*req), M_UFSHCI, how | M_ZERO); 529 if (req != NULL) { 530 req->cb_fn = cb_fn; 531 req->cb_arg = cb_arg; 532 } 533 return (req); 534 } 535 536 static __inline struct ufshci_request * 537 ufshci_allocate_request_vaddr(void *payload, uint32_t payload_size, 538 const int how, ufshci_cb_fn_t cb_fn, void *cb_arg) 539 { 540 struct ufshci_request *req; 541 542 req = _ufshci_allocate_request(how, cb_fn, cb_arg); 543 if (req != NULL) { 544 if (payload_size) { 545 req->payload = memdesc_vaddr(payload, payload_size); 546 req->payload_valid = true; 547 } 548 } 549 return (req); 550 } 551 552 static __inline struct ufshci_request * 553 ufshci_allocate_request_bio(struct bio *bio, const int how, 554 ufshci_cb_fn_t cb_fn, void *cb_arg) 555 { 556 struct ufshci_request *req; 557 558 req = _ufshci_allocate_request(how, cb_fn, cb_arg); 559 if (req != NULL) { 560 req->payload = memdesc_bio(bio); 561 req->payload_valid = true; 562 } 563 return (req); 564 } 565 566 #define ufshci_free_request(req) free(req, M_UFSHCI) 567 568 void ufshci_ctrlr_shared_handler(void *arg); 569 570 #endif /* __UFSHCI_PRIVATE_H__ */ 571