8 static inline void *ublk_get_commit_buf(struct ublk_thread *t,  in ublk_get_commit_buf()  argument
13 	if (buf_idx < t->commit_buf_start ||  in ublk_get_commit_buf()
14 			buf_idx >= t->commit_buf_start + t->nr_commit_buf)  in ublk_get_commit_buf()
16 	idx = buf_idx - t->commit_buf_start;  in ublk_get_commit_buf()
17 	return t->commit_buf + idx * t->commit_buf_size;  in ublk_get_commit_buf()
25 static inline unsigned short ublk_alloc_commit_buf(struct ublk_thread *t)  in ublk_alloc_commit_buf()  argument
27 	int idx = allocator_get(&t->commit_buf_alloc);  in ublk_alloc_commit_buf()
30 		return  idx + t->commit_buf_start;  in ublk_alloc_commit_buf()
38 static inline void ublk_free_commit_buf(struct ublk_thread *t,  in ublk_free_commit_buf()  argument
41 	unsigned short idx = i - t->commit_buf_start;  in ublk_free_commit_buf()
43 	ublk_assert(idx < t->nr_commit_buf);  in ublk_free_commit_buf()
44 	ublk_assert(allocator_get_val(&t->commit_buf_alloc, idx) != 0);  in ublk_free_commit_buf()
46 	allocator_put(&t->commit_buf_alloc, idx);  in ublk_free_commit_buf()
59 static unsigned ublk_commit_buf_size(struct ublk_thread *t)  in ublk_commit_buf_size()  argument
61 	struct ublk_dev *dev = t->dev;  in ublk_commit_buf_size()
69 static void free_batch_commit_buf(struct ublk_thread *t)  in free_batch_commit_buf()  argument
71 	if (t->commit_buf) {  in free_batch_commit_buf()
72 		unsigned buf_size = ublk_commit_buf_size(t);  in free_batch_commit_buf()
73 		unsigned int total = buf_size * t->nr_commit_buf;  in free_batch_commit_buf()
75 		munlock(t->commit_buf, total);  in free_batch_commit_buf()
76 		free(t->commit_buf);  in free_batch_commit_buf()
78 	allocator_deinit(&t->commit_buf_alloc);  in free_batch_commit_buf()
79 	free(t->commit);  in free_batch_commit_buf()
82 static int alloc_batch_commit_buf(struct ublk_thread *t)  in alloc_batch_commit_buf()  argument
84 	unsigned buf_size = ublk_commit_buf_size(t);  in alloc_batch_commit_buf()
85 	unsigned int total = buf_size * t->nr_commit_buf;  in alloc_batch_commit_buf()
90 	t->commit = calloc(t->nr_queues, sizeof(*t->commit));  in alloc_batch_commit_buf()
91 	for (i = 0; i < t->dev->dev_info.nr_hw_queues; i++) {  in alloc_batch_commit_buf()
92 		if (t->q_map[i])  in alloc_batch_commit_buf()
93 			t->commit[j++].q_id = i;  in alloc_batch_commit_buf()
96 	allocator_init(&t->commit_buf_alloc, t->nr_commit_buf);  in alloc_batch_commit_buf()
98 	t->commit_buf = NULL;  in alloc_batch_commit_buf()
103 	t->commit_buf = buf;  in alloc_batch_commit_buf()
106 	if (mlock(t->commit_buf, total))  in alloc_batch_commit_buf()
113 	free_batch_commit_buf(t);  in alloc_batch_commit_buf()
117 static unsigned int ublk_thread_nr_queues(const struct ublk_thread *t)  in ublk_thread_nr_queues()  argument
122 	for (i = 0; i < t->dev->dev_info.nr_hw_queues; i++)  in ublk_thread_nr_queues()
123 		ret += !!t->q_map[i];  in ublk_thread_nr_queues()
128 void ublk_batch_prepare(struct ublk_thread *t)  in ublk_batch_prepare()  argument
138 	struct ublk_queue *q = &t->dev->q[0];  in ublk_batch_prepare()
141 	t->nr_queues = ublk_thread_nr_queues(t);  in ublk_batch_prepare()
143 	t->commit_buf_elem_size = ublk_commit_elem_buf_size(t->dev);  in ublk_batch_prepare()
144 	t->commit_buf_size = ublk_commit_buf_size(t);  in ublk_batch_prepare()
145 	t->commit_buf_start = t->nr_bufs;  in ublk_batch_prepare()
146 	t->nr_commit_buf = 2 * t->nr_queues;  in ublk_batch_prepare()
147 	t->nr_bufs += t->nr_commit_buf;  in ublk_batch_prepare()
149 	t->cmd_flags = 0;  in ublk_batch_prepare()
152 			t->cmd_flags |= UBLK_BATCH_F_AUTO_BUF_REG_FALLBACK;  in ublk_batch_prepare()
154 		t->cmd_flags |= UBLK_BATCH_F_HAS_BUF_ADDR;  in ublk_batch_prepare()
156 	t->state |= UBLKS_T_BATCH_IO;  in ublk_batch_prepare()
159 			__func__, t->idx,  in ublk_batch_prepare()
160 			t->nr_commit_buf, t->commit_buf_size,  in ublk_batch_prepare()
161 			t->nr_bufs);  in ublk_batch_prepare()
164 static void free_batch_fetch_buf(struct ublk_thread *t)  in free_batch_fetch_buf()  argument
168 	for (i = 0; i < t->nr_fetch_bufs; i++) {  in free_batch_fetch_buf()
169 		io_uring_free_buf_ring(&t->ring, t->fetch[i].br, 1, i);  in free_batch_fetch_buf()
170 		munlock(t->fetch[i].fetch_buf, t->fetch[i].fetch_buf_size);  in free_batch_fetch_buf()
171 		free(t->fetch[i].fetch_buf);  in free_batch_fetch_buf()
173 	free(t->fetch);  in free_batch_fetch_buf()
176 static int alloc_batch_fetch_buf(struct ublk_thread *t)  in alloc_batch_fetch_buf()  argument
180 	unsigned buf_size = round_up(t->dev->dev_info.queue_depth * 2, pg_sz);  in alloc_batch_fetch_buf()
185 	t->nr_fetch_bufs = t->nr_queues * 2;  in alloc_batch_fetch_buf()
186 	t->fetch = calloc(t->nr_fetch_bufs, sizeof(*t->fetch));  in alloc_batch_fetch_buf()
189 	for (i = 0; i < t->nr_fetch_bufs; i++) {  in alloc_batch_fetch_buf()
190 		t->fetch[i].fetch_buf_size = buf_size;  in alloc_batch_fetch_buf()
192 		if (posix_memalign((void **)&t->fetch[i].fetch_buf, pg_sz,  in alloc_batch_fetch_buf()
193 					t->fetch[i].fetch_buf_size))  in alloc_batch_fetch_buf()
197 		if (mlock(t->fetch[i].fetch_buf, t->fetch[i].fetch_buf_size))  in alloc_batch_fetch_buf()
200 		t->fetch[i].br = io_uring_setup_buf_ring(&t->ring, 1,  in alloc_batch_fetch_buf()
202 		if (!t->fetch[i].br) {  in alloc_batch_fetch_buf()
211 int ublk_batch_alloc_buf(struct ublk_thread *t)  in ublk_batch_alloc_buf()  argument
215 	ublk_assert(t->nr_commit_buf < 2 * UBLK_MAX_QUEUES);  in ublk_batch_alloc_buf()
217 	ret = alloc_batch_commit_buf(t);  in ublk_batch_alloc_buf()
220 	return alloc_batch_fetch_buf(t);  in ublk_batch_alloc_buf()
223 void ublk_batch_free_buf(struct ublk_thread *t)  in ublk_batch_free_buf()  argument
225 	free_batch_commit_buf(t);  in ublk_batch_free_buf()
226 	free_batch_fetch_buf(t);  in ublk_batch_free_buf()
229 static void ublk_init_batch_cmd(struct ublk_thread *t, __u16 q_id,  in ublk_init_batch_cmd()  argument
255 	t->cmd_inflight += 1;  in ublk_init_batch_cmd()
260 			__func__, t->idx, q_id, op, user_data,  in ublk_init_batch_cmd()
262 			nr_elem * elem_bytes, buf_idx, t->cmd_inflight);  in ublk_init_batch_cmd()
265 static void ublk_setup_commit_sqe(struct ublk_thread *t,  in ublk_setup_commit_sqe()  argument
274 	cmd->flags |= t->cmd_flags;  in ublk_setup_commit_sqe()
277 static void ublk_batch_queue_fetch(struct ublk_thread *t,  in ublk_batch_queue_fetch()  argument
281 	unsigned short nr_elem = t->fetch[buf_idx].fetch_buf_size / 2;  in ublk_batch_queue_fetch()
284 	io_uring_buf_ring_add(t->fetch[buf_idx].br, t->fetch[buf_idx].fetch_buf,  in ublk_batch_queue_fetch()
285 			t->fetch[buf_idx].fetch_buf_size,  in ublk_batch_queue_fetch()
287 	io_uring_buf_ring_advance(t->fetch[buf_idx].br, 1);  in ublk_batch_queue_fetch()
289 	ublk_io_alloc_sqes(t, &sqe, 1);  in ublk_batch_queue_fetch()
291 	ublk_init_batch_cmd(t, q->q_id, sqe, UBLK_U_IO_FETCH_IO_CMDS, 2, nr_elem,  in ublk_batch_queue_fetch()
298 	t->fetch[buf_idx].fetch_buf_off = 0;  in ublk_batch_queue_fetch()
301 void ublk_batch_start_fetch(struct ublk_thread *t)  in ublk_batch_start_fetch()  argument
306 	for (i = 0; i < t->dev->dev_info.nr_hw_queues; i++) {  in ublk_batch_start_fetch()
307 		if (t->q_map[i]) {  in ublk_batch_start_fetch()
308 			struct ublk_queue *q = &t->dev->q[i];  in ublk_batch_start_fetch()
311 			ublk_batch_queue_fetch(t, q, j++);  in ublk_batch_start_fetch()
312 			ublk_batch_queue_fetch(t, q, j++);  in ublk_batch_start_fetch()
317 static unsigned short ublk_compl_batch_fetch(struct ublk_thread *t,  in ublk_compl_batch_fetch()  argument
322 	unsigned start = t->fetch[buf_idx].fetch_buf_off;  in ublk_compl_batch_fetch()
324 	void *buf = t->fetch[buf_idx].fetch_buf;  in ublk_compl_batch_fetch()
348 			q->tgt_ops->queue_io(t, q, tag);  in ublk_compl_batch_fetch()
350 	t->fetch[buf_idx].fetch_buf_off = end;  in ublk_compl_batch_fetch()
354 static int __ublk_batch_queue_prep_io_cmds(struct ublk_thread *t, struct ublk_queue *q)  in __ublk_batch_queue_prep_io_cmds()  argument
357 	unsigned short buf_idx = ublk_alloc_commit_buf(t);  in __ublk_batch_queue_prep_io_cmds()
364 	ublk_io_alloc_sqes(t, &sqe, 1);  in __ublk_batch_queue_prep_io_cmds()
367 	buf = ublk_get_commit_buf(t, buf_idx);  in __ublk_batch_queue_prep_io_cmds()
370 				buf + i * t->commit_buf_elem_size);  in __ublk_batch_queue_prep_io_cmds()
377 			elem->buf_index = ublk_batch_io_buf_idx(t, q, i);  in __ublk_batch_queue_prep_io_cmds()
383 	sqe->len = t->commit_buf_elem_size * nr_elem;  in __ublk_batch_queue_prep_io_cmds()
385 	ublk_init_batch_cmd(t, q->q_id, sqe, UBLK_U_IO_PREP_IO_CMDS,  in __ublk_batch_queue_prep_io_cmds()
386 			t->commit_buf_elem_size, nr_elem, buf_idx);  in __ublk_batch_queue_prep_io_cmds()
387 	ublk_setup_commit_sqe(t, sqe, buf_idx);  in __ublk_batch_queue_prep_io_cmds()
391 int ublk_batch_queue_prep_io_cmds(struct ublk_thread *t, struct ublk_queue *q)  in ublk_batch_queue_prep_io_cmds()  argument
398 	ret = __ublk_batch_queue_prep_io_cmds(t, q);  in ublk_batch_queue_prep_io_cmds()
407 static void ublk_batch_compl_commit_cmd(struct ublk_thread *t,  in ublk_batch_compl_commit_cmd()  argument
418 		ublk_assert(cqe->res == t->commit_buf_elem_size * nr_elem);  in ublk_batch_compl_commit_cmd()
422 	ublk_free_commit_buf(t, buf_idx);  in ublk_batch_compl_commit_cmd()
425 void ublk_batch_compl_cmd(struct ublk_thread *t,  in ublk_batch_compl_cmd()  argument
435 		t->cmd_inflight--;  in ublk_batch_compl_cmd()
436 		ublk_batch_compl_commit_cmd(t, cqe, op);  in ublk_batch_compl_cmd()
442 	q = &t->dev->q[q_id];  in ublk_batch_compl_cmd()
443 	buf_idx = ublk_compl_batch_fetch(t, q, cqe);  in ublk_batch_compl_cmd()
446 		t->cmd_inflight--;  in ublk_batch_compl_cmd()
447 		t->state |= UBLKS_T_STOPPING;  in ublk_batch_compl_cmd()
449 		t->cmd_inflight--;  in ublk_batch_compl_cmd()
450 		ublk_batch_queue_fetch(t, q, buf_idx);  in ublk_batch_compl_cmd()
454 static void __ublk_batch_commit_io_cmds(struct ublk_thread *t,  in __ublk_batch_commit_io_cmds()  argument
463 		ublk_free_commit_buf(t, cb->buf_idx);  in __ublk_batch_commit_io_cmds()
467 	ublk_io_alloc_sqes(t, &sqe, 1);  in __ublk_batch_commit_io_cmds()
470 	sqe->len = nr_elem * t->commit_buf_elem_size;  in __ublk_batch_commit_io_cmds()
473 	ublk_init_batch_cmd(t, cb->q_id, sqe, UBLK_U_IO_COMMIT_IO_CMDS,  in __ublk_batch_commit_io_cmds()
474 			t->commit_buf_elem_size, nr_elem, buf_idx);  in __ublk_batch_commit_io_cmds()
475 	ublk_setup_commit_sqe(t, sqe, buf_idx);  in __ublk_batch_commit_io_cmds()
478 void ublk_batch_commit_io_cmds(struct ublk_thread *t)  in ublk_batch_commit_io_cmds()  argument
482 	for (i = 0; i < t->nr_queues; i++) {  in ublk_batch_commit_io_cmds()
483 		struct batch_commit_buf *cb = &t->commit[i];  in ublk_batch_commit_io_cmds()
486 			__ublk_batch_commit_io_cmds(t, cb);  in ublk_batch_commit_io_cmds()
491 static void __ublk_batch_init_commit(struct ublk_thread *t,  in __ublk_batch_init_commit()  argument
497 	cb->elem = ublk_get_commit_buf(t, buf_idx);  in __ublk_batch_init_commit()
499 	cb->count = t->commit_buf_size /  in __ublk_batch_init_commit()
500 		t->commit_buf_elem_size;  in __ublk_batch_init_commit()
504 static void ublk_batch_init_commit(struct ublk_thread *t,  in ublk_batch_init_commit()  argument
507 	unsigned short buf_idx = ublk_alloc_commit_buf(t);  in ublk_batch_init_commit()
512 	__ublk_batch_init_commit(t, cb, buf_idx);  in ublk_batch_init_commit()
515 void ublk_batch_prep_commit(struct ublk_thread *t)  in ublk_batch_prep_commit()  argument
519 	for (i = 0; i < t->nr_queues; i++)  in ublk_batch_prep_commit()
520 		t->commit[i].buf_idx = UBLKS_T_COMMIT_BUF_INV_IDX;  in ublk_batch_prep_commit()
523 void ublk_batch_complete_io(struct ublk_thread *t, struct ublk_queue *q,  in ublk_batch_complete_io()  argument
526 	unsigned q_t_idx = ublk_queue_idx_in_thread(t, q);  in ublk_batch_complete_io()
527 	struct batch_commit_buf *cb = &t->commit[q_t_idx];  in ublk_batch_complete_io()
532 		ublk_batch_init_commit(t, cb);  in ublk_batch_complete_io()
536 	elem = (struct ublk_batch_elem *)(cb->elem + cb->done * t->commit_buf_elem_size);  in ublk_batch_complete_io()
538 	elem->buf_index = ublk_batch_io_buf_idx_next(t, q, tag);  in ublk_batch_complete_io()