Lines Matching +full:t +full:- +full:head
1 // SPDX-License-Identifier: GPL-2.0
20 /* BIDs are addressed by a 16-bit field in a CQE */
23 /* Mapped buffer ring, return io_uring_buf from head */
24 #define io_ring_head_to_buf(br, head, mask) &(br)->bufs[(head) & (mask)] argument
41 buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask); in io_kbuf_inc_commit()
42 this_len = min_t(int, len, buf->len); in io_kbuf_inc_commit()
43 buf->len -= this_len; in io_kbuf_inc_commit()
44 if (buf->len) { in io_kbuf_inc_commit()
45 buf->addr += this_len; in io_kbuf_inc_commit()
48 bl->head++; in io_kbuf_inc_commit()
49 len -= this_len; in io_kbuf_inc_commit()
57 if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT))) in io_kbuf_commit()
60 req->flags &= ~REQ_F_BUFFERS_COMMIT; in io_kbuf_commit()
64 if (bl->flags & IOBL_INC) in io_kbuf_commit()
66 bl->head += nr; in io_kbuf_commit()
73 lockdep_assert_held(&ctx->uring_lock); in io_buffer_get_list()
75 return xa_load(&ctx->io_bl_xa, bgid); in io_buffer_get_list()
83 * The normal lookup doesn't care about the visibility as we're in io_buffer_add_list()
84 * always under the ->uring_lock, but lookups from mmap do. in io_buffer_add_list()
86 bl->bgid = bgid; in io_buffer_add_list()
87 guard(mutex)(&ctx->mmap_lock); in io_buffer_add_list()
88 return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL)); in io_buffer_add_list()
93 if (WARN_ON_ONCE(!(req->flags & REQ_F_BUFFER_SELECTED))) in io_kbuf_drop_legacy()
95 req->flags &= ~REQ_F_BUFFER_SELECTED; in io_kbuf_drop_legacy()
96 kfree(req->kbuf); in io_kbuf_drop_legacy()
97 req->kbuf = NULL; in io_kbuf_drop_legacy()
102 struct io_ring_ctx *ctx = req->ctx; in io_kbuf_recycle_legacy()
108 buf = req->kbuf; in io_kbuf_recycle_legacy()
109 bl = io_buffer_get_list(ctx, buf->bgid); in io_kbuf_recycle_legacy()
110 list_add(&buf->list, &bl->buf_list); in io_kbuf_recycle_legacy()
111 bl->nbufs++; in io_kbuf_recycle_legacy()
112 req->flags &= ~REQ_F_BUFFER_SELECTED; in io_kbuf_recycle_legacy()
121 if (!list_empty(&bl->buf_list)) { in io_provided_buffer_select()
124 kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list); in io_provided_buffer_select()
125 list_del(&kbuf->list); in io_provided_buffer_select()
126 bl->nbufs--; in io_provided_buffer_select()
127 if (*len == 0 || *len > kbuf->len) in io_provided_buffer_select()
128 *len = kbuf->len; in io_provided_buffer_select()
129 if (list_empty(&bl->buf_list)) in io_provided_buffer_select()
130 req->flags |= REQ_F_BL_EMPTY; in io_provided_buffer_select()
131 req->flags |= REQ_F_BUFFER_SELECTED; in io_provided_buffer_select()
132 req->kbuf = kbuf; in io_provided_buffer_select()
133 req->buf_index = kbuf->bid; in io_provided_buffer_select()
134 return u64_to_user_ptr(kbuf->addr); in io_provided_buffer_select()
147 return -ENOBUFS; in io_provided_buffers_select()
158 struct io_uring_buf_ring *br = bl->buf_ring; in io_ring_buffer_select()
159 __u16 tail, head = bl->head; in io_ring_buffer_select() local
163 tail = smp_load_acquire(&br->tail); in io_ring_buffer_select()
164 if (unlikely(tail == head)) in io_ring_buffer_select()
167 if (head + 1 == tail) in io_ring_buffer_select()
168 req->flags |= REQ_F_BL_EMPTY; in io_ring_buffer_select()
170 buf = io_ring_head_to_buf(br, head, bl->mask); in io_ring_buffer_select()
171 if (*len == 0 || *len > buf->len) in io_ring_buffer_select()
172 *len = buf->len; in io_ring_buffer_select()
173 req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT; in io_ring_buffer_select()
174 req->buf_list = bl; in io_ring_buffer_select()
175 req->buf_index = buf->bid; in io_ring_buffer_select()
176 ret = u64_to_user_ptr(buf->addr); in io_ring_buffer_select()
181 * buffer here, otherwise nothing ensures that the buffer won't in io_ring_buffer_select()
184 * io-wq context and there may be further retries in async hybrid in io_ring_buffer_select()
186 * the transfer completes (or if we get -EAGAIN and must poll of in io_ring_buffer_select()
190 req->buf_list = NULL; in io_ring_buffer_select()
198 struct io_ring_ctx *ctx = req->ctx; in io_buffer_select()
202 io_ring_submit_lock(req->ctx, issue_flags); in io_buffer_select()
206 if (bl->flags & IOBL_BUF_RING) in io_buffer_select()
211 io_ring_submit_unlock(req->ctx, issue_flags); in io_buffer_select()
221 struct io_uring_buf_ring *br = bl->buf_ring; in io_ring_buffers_peek()
222 struct iovec *iov = arg->iovs; in io_ring_buffers_peek()
223 int nr_iovs = arg->nr_iovs; in io_ring_buffers_peek()
224 __u16 nr_avail, tail, head; in io_ring_buffers_peek() local
227 tail = smp_load_acquire(&br->tail); in io_ring_buffers_peek()
228 head = bl->head; in io_ring_buffers_peek()
229 nr_avail = min_t(__u16, tail - head, UIO_MAXIOV); in io_ring_buffers_peek()
231 return -ENOBUFS; in io_ring_buffers_peek()
233 buf = io_ring_head_to_buf(br, head, bl->mask); in io_ring_buffers_peek()
234 if (arg->max_len) { in io_ring_buffers_peek()
235 u32 len = READ_ONCE(buf->len); in io_ring_buffers_peek()
239 return -ENOBUFS; in io_ring_buffers_peek()
240 needed = (arg->max_len + len - 1) / len; in io_ring_buffers_peek()
250 if (arg->mode & KBUF_MODE_EXPAND && nr_avail > nr_iovs && arg->max_len) { in io_ring_buffers_peek()
253 return -ENOMEM; in io_ring_buffers_peek()
254 if (arg->mode & KBUF_MODE_FREE) in io_ring_buffers_peek()
255 kfree(arg->iovs); in io_ring_buffers_peek()
256 arg->iovs = iov; in io_ring_buffers_peek()
263 if (!arg->max_len) in io_ring_buffers_peek()
264 arg->max_len = INT_MAX; in io_ring_buffers_peek()
266 req->buf_index = buf->bid; in io_ring_buffers_peek()
268 u32 len = buf->len; in io_ring_buffers_peek()
271 if (len > arg->max_len) { in io_ring_buffers_peek()
272 len = arg->max_len; in io_ring_buffers_peek()
273 if (!(bl->flags & IOBL_INC)) { in io_ring_buffers_peek()
274 arg->partial_map = 1; in io_ring_buffers_peek()
275 if (iov != arg->iovs) in io_ring_buffers_peek()
277 buf->len = len; in io_ring_buffers_peek()
281 iov->iov_base = u64_to_user_ptr(buf->addr); in io_ring_buffers_peek()
282 iov->iov_len = len; in io_ring_buffers_peek()
285 arg->out_len += len; in io_ring_buffers_peek()
286 arg->max_len -= len; in io_ring_buffers_peek()
287 if (!arg->max_len) in io_ring_buffers_peek()
290 buf = io_ring_head_to_buf(br, ++head, bl->mask); in io_ring_buffers_peek()
291 } while (--nr_iovs); in io_ring_buffers_peek()
293 if (head == tail) in io_ring_buffers_peek()
294 req->flags |= REQ_F_BL_EMPTY; in io_ring_buffers_peek()
296 req->flags |= REQ_F_BUFFER_RING; in io_ring_buffers_peek()
297 req->buf_list = bl; in io_ring_buffers_peek()
298 return iov - arg->iovs; in io_ring_buffers_peek()
304 struct io_ring_ctx *ctx = req->ctx; in io_buffers_select()
306 int ret = -ENOENT; in io_buffers_select()
309 bl = io_buffer_get_list(ctx, arg->buf_group); in io_buffers_select()
313 if (bl->flags & IOBL_BUF_RING) { in io_buffers_select()
316 * Don't recycle these buffers if we need to go through poll. in io_buffers_select()
323 req->flags |= REQ_F_BUFFERS_COMMIT | REQ_F_BL_NO_RECYCLE; in io_buffers_select()
324 io_kbuf_commit(req, bl, arg->out_len, ret); in io_buffers_select()
327 ret = io_provided_buffers_select(req, &arg->out_len, bl, arg->iovs); in io_buffers_select()
336 struct io_ring_ctx *ctx = req->ctx; in io_buffers_peek()
340 lockdep_assert_held(&ctx->uring_lock); in io_buffers_peek()
342 bl = io_buffer_get_list(ctx, arg->buf_group); in io_buffers_peek()
344 return -ENOENT; in io_buffers_peek()
346 if (bl->flags & IOBL_BUF_RING) { in io_buffers_peek()
349 req->flags |= REQ_F_BUFFERS_COMMIT; in io_buffers_peek()
353 /* don't support multiple buffer selections for legacy */ in io_buffers_peek()
354 return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs); in io_buffers_peek()
359 struct io_buffer_list *bl = req->buf_list; in __io_put_kbuf_ring()
365 req->flags &= ~REQ_F_BUFFER_RING; in __io_put_kbuf_ring()
373 ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT); in __io_put_kbufs()
375 if (unlikely(!(req->flags & REQ_F_BUFFER_RING))) { in __io_put_kbufs()
393 lockdep_assert_held(&ctx->uring_lock); in io_remove_buffers_legacy()
394 WARN_ON_ONCE(bl->flags & IOBL_BUF_RING); in io_remove_buffers_legacy()
396 for (i = 0; i < nbufs && !list_empty(&bl->buf_list); i++) { in io_remove_buffers_legacy()
397 nxt = list_first_entry(&bl->buf_list, struct io_buffer, list); in io_remove_buffers_legacy()
398 list_del(&nxt->list); in io_remove_buffers_legacy()
399 bl->nbufs--; in io_remove_buffers_legacy()
408 if (bl->flags & IOBL_BUF_RING) in io_put_bl()
409 io_free_region(ctx, &bl->region); in io_put_bl()
411 io_remove_buffers_legacy(ctx, bl, -1U); in io_put_bl()
423 scoped_guard(mutex, &ctx->mmap_lock) { in io_destroy_buffers()
424 bl = xa_find(&ctx->io_bl_xa, &index, ULONG_MAX, XA_PRESENT); in io_destroy_buffers()
426 xa_erase(&ctx->io_bl_xa, bl->bgid); in io_destroy_buffers()
436 scoped_guard(mutex, &ctx->mmap_lock) in io_destroy_bl()
437 WARN_ON_ONCE(xa_erase(&ctx->io_bl_xa, bl->bgid) != bl); in io_destroy_bl()
446 if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off || in io_remove_buffers_prep()
447 sqe->splice_fd_in) in io_remove_buffers_prep()
448 return -EINVAL; in io_remove_buffers_prep()
450 tmp = READ_ONCE(sqe->fd); in io_remove_buffers_prep()
452 return -EINVAL; in io_remove_buffers_prep()
455 p->nbufs = tmp; in io_remove_buffers_prep()
456 p->bgid = READ_ONCE(sqe->buf_group); in io_remove_buffers_prep()
466 if (sqe->rw_flags || sqe->splice_fd_in) in io_provide_buffers_prep()
467 return -EINVAL; in io_provide_buffers_prep()
469 tmp = READ_ONCE(sqe->fd); in io_provide_buffers_prep()
471 return -E2BIG; in io_provide_buffers_prep()
472 p->nbufs = tmp; in io_provide_buffers_prep()
473 p->addr = READ_ONCE(sqe->addr); in io_provide_buffers_prep()
474 p->len = READ_ONCE(sqe->len); in io_provide_buffers_prep()
475 if (!p->len) in io_provide_buffers_prep()
476 return -EINVAL; in io_provide_buffers_prep()
478 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs, in io_provide_buffers_prep()
480 return -EOVERFLOW; in io_provide_buffers_prep()
481 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check)) in io_provide_buffers_prep()
482 return -EOVERFLOW; in io_provide_buffers_prep()
483 if (!access_ok(u64_to_user_ptr(p->addr), size)) in io_provide_buffers_prep()
484 return -EFAULT; in io_provide_buffers_prep()
486 p->bgid = READ_ONCE(sqe->buf_group); in io_provide_buffers_prep()
487 tmp = READ_ONCE(sqe->off); in io_provide_buffers_prep()
489 return -E2BIG; in io_provide_buffers_prep()
490 if (tmp + p->nbufs > MAX_BIDS_PER_BGID) in io_provide_buffers_prep()
491 return -EINVAL; in io_provide_buffers_prep()
492 p->bid = tmp; in io_provide_buffers_prep()
500 u64 addr = pbuf->addr; in io_add_buffers()
501 int ret = -ENOMEM, i, bid = pbuf->bid; in io_add_buffers()
503 for (i = 0; i < pbuf->nbufs; i++) { in io_add_buffers()
509 if (bl->nbufs == USHRT_MAX) { in io_add_buffers()
510 ret = -EOVERFLOW; in io_add_buffers()
517 list_add_tail(&buf->list, &bl->buf_list); in io_add_buffers()
518 bl->nbufs++; in io_add_buffers()
519 buf->addr = addr; in io_add_buffers()
520 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT); in io_add_buffers()
521 buf->bid = bid; in io_add_buffers()
522 buf->bgid = pbuf->bgid; in io_add_buffers()
523 addr += pbuf->len; in io_add_buffers()
538 if (req->opcode != IORING_OP_PROVIDE_BUFFERS) in __io_manage_buffers_legacy()
539 return -ENOENT; in __io_manage_buffers_legacy()
542 return -ENOMEM; in __io_manage_buffers_legacy()
544 INIT_LIST_HEAD(&bl->buf_list); in __io_manage_buffers_legacy()
545 ret = io_buffer_add_list(req->ctx, bl, p->bgid); in __io_manage_buffers_legacy()
551 /* can't use provide/remove buffers command on mapped buffers */ in __io_manage_buffers_legacy()
552 if (bl->flags & IOBL_BUF_RING) in __io_manage_buffers_legacy()
553 return -EINVAL; in __io_manage_buffers_legacy()
554 if (req->opcode == IORING_OP_PROVIDE_BUFFERS) in __io_manage_buffers_legacy()
555 return io_add_buffers(req->ctx, p, bl); in __io_manage_buffers_legacy()
556 return io_remove_buffers_legacy(req->ctx, bl, p->nbufs); in __io_manage_buffers_legacy()
562 struct io_ring_ctx *ctx = req->ctx; in io_manage_buffers_legacy()
567 bl = io_buffer_get_list(ctx, p->bgid); in io_manage_buffers_legacy()
587 lockdep_assert_held(&ctx->uring_lock); in io_register_pbuf_ring()
590 return -EFAULT; in io_register_pbuf_ring()
592 return -EINVAL; in io_register_pbuf_ring()
594 return -EINVAL; in io_register_pbuf_ring()
596 return -EINVAL; in io_register_pbuf_ring()
597 /* cannot disambiguate full vs empty due to head/tail size */ in io_register_pbuf_ring()
599 return -EINVAL; in io_register_pbuf_ring()
603 /* if mapped buffer ring OR classic exists, don't allow */ in io_register_pbuf_ring()
604 if (bl->flags & IOBL_BUF_RING || !list_empty(&bl->buf_list)) in io_register_pbuf_ring()
605 return -EEXIST; in io_register_pbuf_ring()
611 return -ENOMEM; in io_register_pbuf_ring()
622 ret = io_create_region_mmap_safe(ctx, &bl->region, &rd, mmap_offset); in io_register_pbuf_ring()
625 br = io_region_get_ptr(&bl->region); in io_register_pbuf_ring()
631 * nicely. We cannot do that if IOU_PBUF_RING_MMAP isn't set and in io_register_pbuf_ring()
633 * if we, by chance, don't end up with aligned addresses. The app in io_register_pbuf_ring()
638 ((reg.ring_addr | (unsigned long)br) & (SHM_COLOUR - 1))) { in io_register_pbuf_ring()
639 ret = -EINVAL; in io_register_pbuf_ring()
644 bl->nr_entries = reg.ring_entries; in io_register_pbuf_ring()
645 bl->mask = reg.ring_entries - 1; in io_register_pbuf_ring()
646 bl->flags |= IOBL_BUF_RING; in io_register_pbuf_ring()
647 bl->buf_ring = br; in io_register_pbuf_ring()
649 bl->flags |= IOBL_INC; in io_register_pbuf_ring()
653 io_free_region(ctx, &bl->region); in io_register_pbuf_ring()
663 lockdep_assert_held(&ctx->uring_lock); in io_unregister_pbuf_ring()
666 return -EFAULT; in io_unregister_pbuf_ring()
668 return -EINVAL; in io_unregister_pbuf_ring()
672 return -ENOENT; in io_unregister_pbuf_ring()
673 if (!(bl->flags & IOBL_BUF_RING)) in io_unregister_pbuf_ring()
674 return -EINVAL; in io_unregister_pbuf_ring()
676 scoped_guard(mutex, &ctx->mmap_lock) in io_unregister_pbuf_ring()
677 xa_erase(&ctx->io_bl_xa, bl->bgid); in io_unregister_pbuf_ring()
689 return -EFAULT; in io_register_pbuf_status()
691 return -EINVAL; in io_register_pbuf_status()
695 return -ENOENT; in io_register_pbuf_status()
696 if (!(bl->flags & IOBL_BUF_RING)) in io_register_pbuf_status()
697 return -EINVAL; in io_register_pbuf_status()
699 buf_status.head = bl->head; in io_register_pbuf_status()
701 return -EFAULT; in io_register_pbuf_status()
711 lockdep_assert_held(&ctx->mmap_lock); in io_pbuf_get_region()
713 bl = xa_load(&ctx->io_bl_xa, bgid); in io_pbuf_get_region()
714 if (!bl || !(bl->flags & IOBL_BUF_RING)) in io_pbuf_get_region()
716 return &bl->region; in io_pbuf_get_region()