1 /*-
2 * Copyright (c) 2025, Samsung Electronics Co., Ltd.
3 * Written by Jaeyoon Choi
4 *
5 * SPDX-License-Identifier: BSD-2-Clause
6 */
7
8 #ifndef __UFSHCI_PRIVATE_H__
9 #define __UFSHCI_PRIVATE_H__
10
11 #ifdef _KERNEL
12 #include <sys/types.h>
13 #else /* !_KERNEL */
14 #include <stdbool.h>
15 #include <stdint.h>
16 #endif /* _KERNEL */
17
18 #include <sys/param.h>
19 #include <sys/systm.h>
20 #include <sys/bio.h>
21 #include <sys/bus.h>
22 #include <sys/counter.h>
23 #include <sys/kernel.h>
24 #include <sys/lock.h>
25 #include <sys/malloc.h>
26 #include <sys/memdesc.h>
27 #include <sys/module.h>
28 #include <sys/mutex.h>
29 #include <sys/rman.h>
30 #include <sys/taskqueue.h>
31
32 #include <machine/bus.h>
33
34 #include <cam/cam.h>
35
36 #include "ufshci.h"
37
38 MALLOC_DECLARE(M_UFSHCI);
39
40 #define UFSHCI_DEVICE_INIT_TIMEOUT_MS (2000) /* in milliseconds */
41 #define UFSHCI_UIC_CMD_TIMEOUT_MS (500) /* in milliseconds */
42 #define UFSHCI_DEFAULT_TIMEOUT_PERIOD (10) /* in seconds */
43 #define UFSHCI_MIN_TIMEOUT_PERIOD (5) /* in seconds */
44 #define UFSHCI_MAX_TIMEOUT_PERIOD (120) /* in seconds */
45
46 #define UFSHCI_DEFAULT_RETRY_COUNT (4)
47
48 #define UFSHCI_UTR_ENTRIES (32)
49 #define UFSHCI_UTRM_ENTRIES (8)
50
51 #define UFSHCI_SECTOR_SIZE (512)
52
53 struct ufshci_controller;
54
55 struct ufshci_completion_poll_status {
56 struct ufshci_completion cpl;
57 int done;
58 bool error;
59 };
60
61 struct ufshci_request {
62 struct ufshci_upiu request_upiu;
63 size_t request_size;
64 size_t response_size;
65
66 struct memdesc payload;
67 enum ufshci_data_direction data_direction;
68 ufshci_cb_fn_t cb_fn;
69 void *cb_arg;
70 bool is_admin;
71 int32_t retries;
72 bool payload_valid;
73 bool spare[2]; /* Future use */
74 STAILQ_ENTRY(ufshci_request) stailq;
75 };
76
77 enum ufshci_slot_state {
78 UFSHCI_SLOT_STATE_FREE = 0x0,
79 UFSHCI_SLOT_STATE_RESERVED = 0x1,
80 UFSHCI_SLOT_STATE_SCHEDULED = 0x2,
81 UFSHCI_SLOT_STATE_TIMEOUT = 0x3,
82 UFSHCI_SLOT_STATE_NEED_ERROR_HANDLING = 0x4,
83 };
84
85 struct ufshci_tracker {
86 TAILQ_ENTRY(ufshci_tracker) tailq;
87 struct ufshci_request *req;
88 struct ufshci_req_queue *req_queue;
89 struct ufshci_hw_queue *hwq;
90 uint8_t slot_num;
91 enum ufshci_slot_state slot_state;
92 size_t response_size;
93 sbintime_t deadline;
94
95 bus_dmamap_t payload_dma_map;
96 uint64_t payload_addr;
97
98 struct ufshci_utp_cmd_desc *ucd;
99 bus_addr_t ucd_bus_addr;
100
101 uint16_t prdt_off;
102 uint16_t prdt_entry_cnt;
103 };
104
105 enum ufshci_queue_mode {
106 UFSHCI_Q_MODE_SDB = 0x00, /* Single Doorbell Mode*/
107 UFSHCI_Q_MODE_MCQ = 0x01, /* Multi-Circular Queue Mode*/
108 };
109
110 /*
111 * UFS uses slot-based Single Doorbell (SDB) mode for request submission by
112 * default and additionally supports Multi-Circular Queue (MCQ) in UFS 4.0. To
113 * minimize duplicated code between SDB and MCQ, mode dependent operations are
114 * extracted into ufshci_qops.
115 */
116 struct ufshci_qops {
117 int (*construct)(struct ufshci_controller *ctrlr,
118 struct ufshci_req_queue *req_queue, uint32_t num_entries,
119 bool is_task_mgmt);
120 void (*destroy)(struct ufshci_controller *ctrlr,
121 struct ufshci_req_queue *req_queue);
122 struct ufshci_hw_queue *(*get_hw_queue)(
123 struct ufshci_req_queue *req_queue);
124 int (*enable)(struct ufshci_controller *ctrlr,
125 struct ufshci_req_queue *req_queue);
126 void (*disable)(struct ufshci_controller *ctrlr,
127 struct ufshci_req_queue *req_queue);
128 int (*reserve_slot)(struct ufshci_req_queue *req_queue,
129 struct ufshci_tracker **tr);
130 int (*reserve_admin_slot)(struct ufshci_req_queue *req_queue,
131 struct ufshci_tracker **tr);
132 void (*ring_doorbell)(struct ufshci_controller *ctrlr,
133 struct ufshci_tracker *tr);
134 bool (*is_doorbell_cleared)(struct ufshci_controller *ctrlr,
135 uint8_t slot);
136 void (*clear_cpl_ntf)(struct ufshci_controller *ctrlr,
137 struct ufshci_tracker *tr);
138 bool (*process_cpl)(struct ufshci_req_queue *req_queue);
139 int (*get_inflight_io)(struct ufshci_controller *ctrlr);
140 };
141
142 #define UFSHCI_SDB_Q 0 /* Queue number for a single doorbell queue */
143
144 enum ufshci_recovery {
145 RECOVERY_NONE = 0, /* Normal operations */
146 RECOVERY_WAITING, /* waiting for the reset to complete */
147 };
148
149 /*
150 * Generic queue container used by both SDB (fixed 32-slot bitmap) and MCQ
151 * (ring buffer) modes. Fields are shared; some such as sq_head, sq_tail and
152 * cq_head are not used in SDB but used in MCQ.
153 */
154 struct ufshci_hw_queue {
155 struct ufshci_controller *ctrlr;
156 struct ufshci_req_queue *req_queue;
157 uint32_t id;
158 int domain;
159 int cpu;
160
161 struct callout timer; /* recovery lock */
162 bool timer_armed; /* recovery lock */
163 enum ufshci_recovery recovery_state; /* recovery lock */
164
165 union {
166 struct ufshci_utp_xfer_req_desc *utrd;
167 struct ufshci_utp_task_mgmt_req_desc *utmrd;
168 };
169
170 bus_dma_tag_t dma_tag_queue;
171 bus_dmamap_t queuemem_map;
172 bus_addr_t req_queue_addr;
173
174 bus_addr_t *ucd_bus_addr;
175
176 uint32_t num_entries;
177 uint32_t num_trackers;
178
179 TAILQ_HEAD(, ufshci_tracker) free_tr;
180 TAILQ_HEAD(, ufshci_tracker) outstanding_tr;
181
182 /*
183 * A Request List using the single doorbell method uses a dedicated
184 * ufshci_tracker, one per slot.
185 */
186 struct ufshci_tracker **act_tr;
187
188 uint32_t sq_head; /* MCQ mode */
189 uint32_t sq_tail; /* MCQ mode */
190 uint32_t cq_head; /* MCQ mode */
191
192 uint32_t phase;
193 int64_t num_cmds;
194 int64_t num_intr_handler_calls;
195 int64_t num_retries;
196 int64_t num_failures;
197
198 /*
199 * Each lock may be acquired independently.
200 * When both are required, acquire them in this order to avoid
201 * deadlocks. (recovery_lock -> qlock)
202 */
203 struct mtx_padalign qlock;
204 struct mtx_padalign recovery_lock;
205 };
206
207 struct ufshci_req_queue {
208 struct ufshci_controller *ctrlr;
209 int domain;
210
211 /*
212 * queue_mode: active transfer scheme
213 * UFSHCI_Q_MODE_SDB – legacy single‑doorbell list
214 * UFSHCI_Q_MODE_MCQ – modern multi‑circular queue (UFSHCI 4.0+)
215 */
216 enum ufshci_queue_mode queue_mode;
217
218 uint8_t num_q;
219 struct ufshci_hw_queue *hwq;
220
221 struct ufshci_qops qops;
222
223 bool is_task_mgmt;
224 uint32_t num_entries;
225 uint32_t num_trackers;
226
227 /* Shared DMA resource */
228 struct ufshci_utp_cmd_desc *ucd;
229
230 bus_dma_tag_t dma_tag_ucd;
231 bus_dma_tag_t dma_tag_payload;
232
233 bus_dmamap_t ucdmem_map;
234 };
235
236 struct ufshci_device {
237 uint32_t max_lun_count;
238
239 struct ufshci_device_descriptor dev_desc;
240 struct ufshci_geometry_descriptor geo_desc;
241
242 uint32_t unipro_version;
243
244 /* WriteBooster */
245 bool is_wb_enabled;
246 bool is_wb_flush_enabled;
247 uint32_t wb_buffer_type;
248 uint32_t wb_buffer_size_mb;
249 uint32_t wb_user_space_config_option;
250 uint8_t wb_dedicated_lu;
251 uint32_t write_booster_flush_threshold;
252
253 /* Power mode */
254 bool power_mode_supported;
255 };
256
257 /*
258 * One of these per allocated device.
259 */
260 struct ufshci_controller {
261 device_t dev;
262
263 uint32_t quirks;
264 #define UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE \
265 1 /* QEMU does not support UIC POWER MODE */
266 #define UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE \
267 2 /* Need an additional 200 ms of PA_TActivate */
268 #define UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE \
269 4 /* Need to wait 1250us after power mode change */
270 #define UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY \
271 8 /* Need to change the number of lanes before changing HS-GEAR. */
272 #define UFSHCI_QUIRK_NOT_SUPPORT_ABORT_TASK \
273 16 /* QEMU does not support Task Management Request */
274 #define UFSHCI_QUIRK_SKIP_WELL_KNOWN_LUNS \
275 32 /* QEMU does not support Well known logical units*/
276
277 uint32_t ref_clk;
278
279 struct cam_sim *ufshci_sim;
280 struct cam_path *ufshci_path;
281
282 struct cam_periph *ufs_device_wlun_periph;
283 struct mtx ufs_device_wlun_mtx;
284
285 struct mtx sc_mtx;
286 uint32_t sc_unit;
287 uint8_t sc_name[16];
288
289 struct ufshci_device ufs_dev;
290
291 bus_space_tag_t bus_tag;
292 bus_space_handle_t bus_handle;
293 int resource_id;
294 struct resource *resource;
295
296 /* Currently, there is no UFSHCI that supports MSI, MSI-X. */
297 int msi_count;
298
299 /* Fields for tracking progress during controller initialization. */
300 struct intr_config_hook config_hook;
301
302 struct task reset_task;
303 struct taskqueue *taskqueue;
304
305 /* For shared legacy interrupt. */
306 int rid;
307 struct resource *res;
308 void *tag;
309
310 uint32_t major_version;
311 uint32_t minor_version;
312
313 uint32_t enable_aborts;
314
315 uint32_t num_io_queues;
316 uint32_t max_hw_pend_io;
317
318 /* Maximum logical unit number */
319 uint32_t max_lun_count;
320
321 /* Maximum i/o size in bytes */
322 uint32_t max_xfer_size;
323
324 /* Controller capacity */
325 uint32_t cap;
326
327 /* Page size and log2(page_size) - 12 that we're currently using */
328 uint32_t page_size;
329
330 /* Timeout value on device initialization */
331 uint32_t device_init_timeout_in_ms;
332
333 /* Timeout value on UIC command */
334 uint32_t uic_cmd_timeout_in_ms;
335
336 /* UTMR/UTR queue timeout period in seconds */
337 uint32_t timeout_period;
338
339 /* UTMR/UTR queue retry count */
340 uint32_t retry_count;
341
342 /* UFS Host Controller Interface Registers */
343 struct ufshci_registers *regs;
344
345 /* UFS Transport Protocol Layer (UTP) */
346 struct ufshci_req_queue task_mgmt_req_queue;
347 struct ufshci_req_queue transfer_req_queue;
348 bool is_single_db_supported; /* 0 = supported */
349 bool is_mcq_supported; /* 1 = supported */
350
351 /* UFS Interconnect Layer (UIC) */
352 struct mtx uic_cmd_lock;
353 uint32_t unipro_version;
354 uint8_t hs_gear;
355 uint32_t tx_lanes;
356 uint32_t rx_lanes;
357 uint32_t max_rx_hs_gear;
358 uint32_t max_tx_lanes;
359 uint32_t max_rx_lanes;
360
361 bool is_failed;
362 };
363
364 #define ufshci_mmio_offsetof(reg) offsetof(struct ufshci_registers, reg)
365
366 #define ufshci_mmio_read_4(sc, reg) \
367 bus_space_read_4((sc)->bus_tag, (sc)->bus_handle, \
368 ufshci_mmio_offsetof(reg))
369
370 #define ufshci_mmio_write_4(sc, reg, val) \
371 bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
372 ufshci_mmio_offsetof(reg), val)
373
374 #define ufshci_printf(ctrlr, fmt, args...) \
375 device_printf(ctrlr->dev, fmt, ##args)
376
377 /* UFSHCI */
378 void ufshci_completion_poll_cb(void *arg, const struct ufshci_completion *cpl,
379 bool error);
380
381 /* SIM */
382 uint8_t ufshci_sim_translate_scsi_to_ufs_lun(lun_id_t scsi_lun);
383 uint64_t ufshci_sim_translate_ufs_to_scsi_lun(uint8_t ufs_lun);
384 int ufshci_sim_attach(struct ufshci_controller *ctrlr);
385 void ufshci_sim_detach(struct ufshci_controller *ctrlr);
386 struct cam_periph *ufshci_sim_find_periph(struct ufshci_controller *ctrlr,
387 uint8_t wlun);
388 int ufshci_sim_send_ssu(struct ufshci_controller *ctrlr, bool start,
389 int pwr_cond, bool immed);
390
391 /* Controller */
392 int ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev);
393 void ufshci_ctrlr_destruct(struct ufshci_controller *ctrlr, device_t dev);
394 void ufshci_ctrlr_reset(struct ufshci_controller *ctrlr);
395 /* ctrlr defined as void * to allow use with config_intrhook. */
396 void ufshci_ctrlr_start_config_hook(void *arg);
397 void ufshci_ctrlr_poll(struct ufshci_controller *ctrlr);
398
399 int ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr,
400 struct ufshci_request *req);
401 int ufshci_ctrlr_submit_admin_request(struct ufshci_controller *ctrlr,
402 struct ufshci_request *req);
403 int ufshci_ctrlr_submit_io_request(struct ufshci_controller *ctrlr,
404 struct ufshci_request *req);
405 int ufshci_ctrlr_send_nop(struct ufshci_controller *ctrlr);
406
407 void ufshci_reg_dump(struct ufshci_controller *ctrlr);
408
409 /* Device */
410 int ufshci_dev_init(struct ufshci_controller *ctrlr);
411 int ufshci_dev_reset(struct ufshci_controller *ctrlr);
412 int ufshci_dev_init_reference_clock(struct ufshci_controller *ctrlr);
413 int ufshci_dev_init_unipro(struct ufshci_controller *ctrlr);
414 int ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr);
415 int ufshci_dev_init_ufs_power_mode(struct ufshci_controller *ctrlr);
416 int ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr);
417 int ufshci_dev_config_write_booster(struct ufshci_controller *ctrlr);
418
419 /* Controller Command */
420 void ufshci_ctrlr_cmd_send_task_mgmt_request(struct ufshci_controller *ctrlr,
421 ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t function, uint8_t lun,
422 uint8_t task_tag, uint8_t iid);
423 void ufshci_ctrlr_cmd_send_nop(struct ufshci_controller *ctrlr,
424 ufshci_cb_fn_t cb_fn, void *cb_arg);
425 void ufshci_ctrlr_cmd_send_query_request(struct ufshci_controller *ctrlr,
426 ufshci_cb_fn_t cb_fn, void *cb_arg, struct ufshci_query_param param);
427 void ufshci_ctrlr_cmd_send_scsi_command(struct ufshci_controller *ctrlr,
428 ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t *cmd_ptr, uint8_t cmd_len,
429 uint32_t data_len, uint8_t lun, bool is_write);
430
431 /* Request Queue */
432 bool ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue);
433 int ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr);
434 int ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr);
435 void ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr);
436 void ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr);
437 void ufshci_utmr_req_queue_disable(struct ufshci_controller *ctrlr);
438 int ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr);
439 void ufshci_utr_req_queue_disable(struct ufshci_controller *ctrlr);
440 int ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr);
441 void ufshci_req_queue_fail(struct ufshci_controller *ctrlr,
442 struct ufshci_hw_queue *hwq);
443 int ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
444 struct ufshci_request *req, bool is_admin);
445 void ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr);
446
447 /* Request Single Doorbell Queue */
448 int ufshci_req_sdb_construct(struct ufshci_controller *ctrlr,
449 struct ufshci_req_queue *req_queue, uint32_t num_entries,
450 bool is_task_mgmt);
451 void ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr,
452 struct ufshci_req_queue *req_queue);
453 struct ufshci_hw_queue *ufshci_req_sdb_get_hw_queue(
454 struct ufshci_req_queue *req_queue);
455 void ufshci_req_sdb_disable(struct ufshci_controller *ctrlr,
456 struct ufshci_req_queue *req_queue);
457 int ufshci_req_sdb_enable(struct ufshci_controller *ctrlr,
458 struct ufshci_req_queue *req_queue);
459 int ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue,
460 struct ufshci_tracker **tr);
461 void ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr,
462 struct ufshci_tracker *tr);
463 void ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr,
464 struct ufshci_tracker *tr);
465 bool ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
466 uint8_t slot);
467 bool ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr,
468 uint8_t slot);
469 void ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr,
470 struct ufshci_tracker *tr);
471 void ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller *ctrlr,
472 struct ufshci_tracker *tr);
473 bool ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue);
474 int ufshci_req_sdb_get_inflight_io(struct ufshci_controller *ctrlr);
475
476 /* UIC Command */
477 int ufshci_uic_power_mode_ready(struct ufshci_controller *ctrlr);
478 int ufshci_uic_cmd_ready(struct ufshci_controller *ctrlr);
479 int ufshci_uic_send_dme_link_startup(struct ufshci_controller *ctrlr);
480 int ufshci_uic_send_dme_get(struct ufshci_controller *ctrlr, uint16_t attribute,
481 uint32_t *return_value);
482 int ufshci_uic_send_dme_set(struct ufshci_controller *ctrlr, uint16_t attribute,
483 uint32_t value);
484 int ufshci_uic_send_dme_peer_get(struct ufshci_controller *ctrlr,
485 uint16_t attribute, uint32_t *return_value);
486 int ufshci_uic_send_dme_peer_set(struct ufshci_controller *ctrlr,
487 uint16_t attribute, uint32_t value);
488 int ufshci_uic_send_dme_endpoint_reset(struct ufshci_controller *ctrlr);
489
490 /* SYSCTL */
491 void ufshci_sysctl_initialize_ctrlr(struct ufshci_controller *ctrlr);
492
493 int ufshci_attach(device_t dev);
494 int ufshci_detach(device_t dev);
495
496 /*
497 * Wait for a command to complete using the ufshci_completion_poll_cb. Used in
498 * limited contexts where the caller knows it's OK to block briefly while the
499 * command runs. The ISR will run the callback which will set status->done to
500 * true, usually within microseconds. If not, then after one second timeout
501 * handler should reset the controller and abort all outstanding requests
502 * including this polled one. If still not after ten seconds, then something is
503 * wrong with the driver, and panic is the only way to recover.
504 *
505 * Most commands using this interface aren't actual I/O to the drive's media so
506 * complete within a few microseconds. Adaptively spin for one tick to catch the
507 * vast majority of these without waiting for a tick plus scheduling delays.
508 * Since these are on startup, this drastically reduces startup time.
509 */
510 static __inline void
ufshci_completion_poll(struct ufshci_completion_poll_status * status)511 ufshci_completion_poll(struct ufshci_completion_poll_status *status)
512 {
513 int timeout = ticks + 10 * hz;
514 sbintime_t delta_t = SBT_1US;
515
516 while (!atomic_load_acq_int(&status->done)) {
517 if (timeout - ticks < 0)
518 panic(
519 "UFSHCI polled command failed to complete within 10s.");
520 pause_sbt("ufshci_cpl", delta_t, 0, C_PREL(1));
521 delta_t = min(SBT_1MS, delta_t * 3 / 2);
522 }
523 }
524
525 static __inline void
ufshci_single_map(void * arg,bus_dma_segment_t * seg,int nseg,int error)526 ufshci_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
527 {
528 uint64_t *bus_addr = (uint64_t *)arg;
529
530 KASSERT(nseg == 1, ("number of segments (%d) is not 1", nseg));
531 if (error != 0)
532 printf("ufshci_single_map err %d\n", error);
533 *bus_addr = seg[0].ds_addr;
534 }
535
536 static __inline struct ufshci_request *
_ufshci_allocate_request(const int how,ufshci_cb_fn_t cb_fn,void * cb_arg)537 _ufshci_allocate_request(const int how, ufshci_cb_fn_t cb_fn, void *cb_arg)
538 {
539 struct ufshci_request *req;
540
541 KASSERT(how == M_WAITOK || how == M_NOWAIT,
542 ("ufshci_allocate_request: invalid how %d", how));
543
544 req = malloc(sizeof(*req), M_UFSHCI, how | M_ZERO);
545 if (req != NULL) {
546 req->cb_fn = cb_fn;
547 req->cb_arg = cb_arg;
548 }
549 return (req);
550 }
551
552 static __inline struct ufshci_request *
ufshci_allocate_request_vaddr(void * payload,uint32_t payload_size,const int how,ufshci_cb_fn_t cb_fn,void * cb_arg)553 ufshci_allocate_request_vaddr(void *payload, uint32_t payload_size,
554 const int how, ufshci_cb_fn_t cb_fn, void *cb_arg)
555 {
556 struct ufshci_request *req;
557
558 req = _ufshci_allocate_request(how, cb_fn, cb_arg);
559 if (req != NULL) {
560 if (payload_size) {
561 req->payload = memdesc_vaddr(payload, payload_size);
562 req->payload_valid = true;
563 }
564 }
565 return (req);
566 }
567
568 static __inline struct ufshci_request *
ufshci_allocate_request_bio(struct bio * bio,const int how,ufshci_cb_fn_t cb_fn,void * cb_arg)569 ufshci_allocate_request_bio(struct bio *bio, const int how,
570 ufshci_cb_fn_t cb_fn, void *cb_arg)
571 {
572 struct ufshci_request *req;
573
574 req = _ufshci_allocate_request(how, cb_fn, cb_arg);
575 if (req != NULL) {
576 req->payload = memdesc_bio(bio);
577 req->payload_valid = true;
578 }
579 return (req);
580 }
581
582 #define ufshci_free_request(req) free(req, M_UFSHCI)
583
584 void ufshci_ctrlr_shared_handler(void *arg);
585
586 #endif /* __UFSHCI_PRIVATE_H__ */
587