13b77495aSJens Axboe // SPDX-License-Identifier: GPL-2.0
23b77495aSJens Axboe #ifndef IOU_KBUF_H
33b77495aSJens Axboe #define IOU_KBUF_H
43b77495aSJens Axboe
53b77495aSJens Axboe #include <uapi/linux/io_uring.h>
63b77495aSJens Axboe
7a69307a5SJens Axboe enum {
8a69307a5SJens Axboe /* ring mapped provided buffers */
9a69307a5SJens Axboe IOBL_BUF_RING = 1,
10a69307a5SJens Axboe /* ring mapped provided buffers, but mmap'ed by application */
11a69307a5SJens Axboe IOBL_MMAP = 2,
12*ae98dbf4SJens Axboe /* buffers are consumed incrementally rather than always fully */
13*ae98dbf4SJens Axboe IOBL_INC = 4,
14*ae98dbf4SJens Axboe
15a69307a5SJens Axboe };
16a69307a5SJens Axboe
173b77495aSJens Axboe struct io_buffer_list {
183b77495aSJens Axboe /*
193b77495aSJens Axboe * If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not,
203b77495aSJens Axboe * then these are classic provided buffers and ->buf_list is used.
213b77495aSJens Axboe */
223b77495aSJens Axboe union {
233b77495aSJens Axboe struct list_head buf_list;
243b77495aSJens Axboe struct {
253b77495aSJens Axboe struct page **buf_pages;
263b77495aSJens Axboe struct io_uring_buf_ring *buf_ring;
273b77495aSJens Axboe };
285cf4f52eSJens Axboe struct rcu_head rcu;
293b77495aSJens Axboe };
303b77495aSJens Axboe __u16 bgid;
313b77495aSJens Axboe
323b77495aSJens Axboe /* below is for ring provided buffers */
333b77495aSJens Axboe __u16 buf_nr_pages;
343b77495aSJens Axboe __u16 nr_entries;
353b77495aSJens Axboe __u16 head;
363b77495aSJens Axboe __u16 mask;
3725a2c188SJens Axboe
38a69307a5SJens Axboe __u16 flags;
396b69c4abSJens Axboe
40a69307a5SJens Axboe atomic_t refs;
413b77495aSJens Axboe };
423b77495aSJens Axboe
433b77495aSJens Axboe struct io_buffer {
443b77495aSJens Axboe struct list_head list;
453b77495aSJens Axboe __u64 addr;
463b77495aSJens Axboe __u32 len;
473b77495aSJens Axboe __u16 bid;
483b77495aSJens Axboe __u16 bgid;
493b77495aSJens Axboe };
503b77495aSJens Axboe
5135c8711cSJens Axboe enum {
5235c8711cSJens Axboe /* can alloc a bigger vec */
5335c8711cSJens Axboe KBUF_MODE_EXPAND = 1,
5435c8711cSJens Axboe /* if bigger vec allocated, free old one */
5535c8711cSJens Axboe KBUF_MODE_FREE = 2,
5635c8711cSJens Axboe };
5735c8711cSJens Axboe
5835c8711cSJens Axboe struct buf_sel_arg {
5935c8711cSJens Axboe struct iovec *iovs;
6035c8711cSJens Axboe size_t out_len;
6135c8711cSJens Axboe size_t max_len;
6212044332SJens Axboe unsigned short nr_iovs;
6312044332SJens Axboe unsigned short mode;
6435c8711cSJens Axboe };
6535c8711cSJens Axboe
663b77495aSJens Axboe void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
673b77495aSJens Axboe unsigned int issue_flags);
6835c8711cSJens Axboe int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
6935c8711cSJens Axboe unsigned int issue_flags);
7035c8711cSJens Axboe int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg);
713b77495aSJens Axboe void io_destroy_buffers(struct io_ring_ctx *ctx);
723b77495aSJens Axboe
733b77495aSJens Axboe int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
743b77495aSJens Axboe int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags);
753b77495aSJens Axboe
763b77495aSJens Axboe int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
773b77495aSJens Axboe int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags);
783b77495aSJens Axboe
793b77495aSJens Axboe int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
803b77495aSJens Axboe int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
81d293b1a8SJens Axboe int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);
823b77495aSJens Axboe
836733e678SJens Axboe void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags);
8453ccf69bSPavel Begunkov
8589d528baSDylan Yudaken bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
86795bbbc8SHao Xu
87561e4f94SJens Axboe void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl);
88561e4f94SJens Axboe struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
89561e4f94SJens Axboe unsigned long bgid);
9087585b05SJens Axboe int io_pbuf_mmap(struct file *file, struct vm_area_struct *vma);
91c56e022cSJens Axboe
io_kbuf_recycle_ring(struct io_kiocb * req)9289d528baSDylan Yudaken static inline bool io_kbuf_recycle_ring(struct io_kiocb *req)
93795bbbc8SHao Xu {
94795bbbc8SHao Xu /*
95795bbbc8SHao Xu * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
96795bbbc8SHao Xu * the flag and hence ensure that bl->head doesn't get incremented.
97795bbbc8SHao Xu * If the tail has already been incremented, hang on to it.
98795bbbc8SHao Xu * The exception is partial io, that case we should increment bl->head
99795bbbc8SHao Xu * to monopolize the buffer.
100795bbbc8SHao Xu */
101795bbbc8SHao Xu if (req->buf_list) {
102795bbbc8SHao Xu req->buf_index = req->buf_list->bgid;
10335c8711cSJens Axboe req->flags &= ~(REQ_F_BUFFER_RING|REQ_F_BUFFERS_COMMIT);
10489d528baSDylan Yudaken return true;
105795bbbc8SHao Xu }
10689d528baSDylan Yudaken return false;
107795bbbc8SHao Xu }
108024b8fdeSHao Xu
io_do_buffer_select(struct io_kiocb * req)1093b77495aSJens Axboe static inline bool io_do_buffer_select(struct io_kiocb *req)
1103b77495aSJens Axboe {
1113b77495aSJens Axboe if (!(req->flags & REQ_F_BUFFER_SELECT))
1123b77495aSJens Axboe return false;
1133b77495aSJens Axboe return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
1143b77495aSJens Axboe }
1153b77495aSJens Axboe
io_kbuf_recycle(struct io_kiocb * req,unsigned issue_flags)11689d528baSDylan Yudaken static inline bool io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
1173b77495aSJens Axboe {
118186daf23SJens Axboe if (req->flags & REQ_F_BL_NO_RECYCLE)
119186daf23SJens Axboe return false;
120024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_SELECTED)
12189d528baSDylan Yudaken return io_kbuf_recycle_legacy(req, issue_flags);
122024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_RING)
12389d528baSDylan Yudaken return io_kbuf_recycle_ring(req);
12489d528baSDylan Yudaken return false;
1253b77495aSJens Axboe }
1263b77495aSJens Axboe
1272c8fa70bSJens Axboe /* Mapped buffer ring, return io_uring_buf from head */
1282c8fa70bSJens Axboe #define io_ring_head_to_buf(br, head, mask) &(br)->bufs[(head) & (mask)]
1292c8fa70bSJens Axboe
io_kbuf_commit(struct io_kiocb * req,struct io_buffer_list * bl,int len,int nr)130*ae98dbf4SJens Axboe static inline bool io_kbuf_commit(struct io_kiocb *req,
1316733e678SJens Axboe struct io_buffer_list *bl, int len, int nr)
132ecd5c9b2SJens Axboe {
133ecd5c9b2SJens Axboe if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT)))
134*ae98dbf4SJens Axboe return true;
135*ae98dbf4SJens Axboe
136ecd5c9b2SJens Axboe req->flags &= ~REQ_F_BUFFERS_COMMIT;
137*ae98dbf4SJens Axboe
138*ae98dbf4SJens Axboe if (unlikely(len < 0))
139*ae98dbf4SJens Axboe return true;
140*ae98dbf4SJens Axboe
141*ae98dbf4SJens Axboe if (bl->flags & IOBL_INC) {
142*ae98dbf4SJens Axboe struct io_uring_buf *buf;
143*ae98dbf4SJens Axboe
144*ae98dbf4SJens Axboe buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask);
145*ae98dbf4SJens Axboe if (WARN_ON_ONCE(len > buf->len))
146*ae98dbf4SJens Axboe len = buf->len;
147*ae98dbf4SJens Axboe buf->len -= len;
148*ae98dbf4SJens Axboe if (buf->len) {
149*ae98dbf4SJens Axboe buf->addr += len;
150*ae98dbf4SJens Axboe return false;
151*ae98dbf4SJens Axboe }
152ecd5c9b2SJens Axboe }
153ecd5c9b2SJens Axboe
154*ae98dbf4SJens Axboe bl->head += nr;
155*ae98dbf4SJens Axboe return true;
156*ae98dbf4SJens Axboe }
157*ae98dbf4SJens Axboe
__io_put_kbuf_ring(struct io_kiocb * req,int len,int nr)158*ae98dbf4SJens Axboe static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
1593b77495aSJens Axboe {
16035c8711cSJens Axboe struct io_buffer_list *bl = req->buf_list;
161*ae98dbf4SJens Axboe bool ret = true;
16235c8711cSJens Axboe
16335c8711cSJens Axboe if (bl) {
164*ae98dbf4SJens Axboe ret = io_kbuf_commit(req, bl, len, nr);
16535c8711cSJens Axboe req->buf_index = bl->bgid;
16632f3c434SDylan Yudaken }
1673b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_RING;
168*ae98dbf4SJens Axboe return ret;
1698435c6f3SJens Axboe }
1708435c6f3SJens Axboe
__io_put_kbuf_list(struct io_kiocb * req,int len,struct list_head * list)1716733e678SJens Axboe static inline void __io_put_kbuf_list(struct io_kiocb *req, int len,
1728435c6f3SJens Axboe struct list_head *list)
1738435c6f3SJens Axboe {
1748435c6f3SJens Axboe if (req->flags & REQ_F_BUFFER_RING) {
1756733e678SJens Axboe __io_put_kbuf_ring(req, len, 1);
1763b77495aSJens Axboe } else {
17732f3c434SDylan Yudaken req->buf_index = req->kbuf->bgid;
1783b77495aSJens Axboe list_add(&req->kbuf->list, list);
1793b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_SELECTED;
1803b77495aSJens Axboe }
1813b77495aSJens Axboe }
1823b77495aSJens Axboe
io_kbuf_drop(struct io_kiocb * req)183bbbef3e9SMing Lei static inline void io_kbuf_drop(struct io_kiocb *req)
1843b77495aSJens Axboe {
1853b77495aSJens Axboe lockdep_assert_held(&req->ctx->completion_lock);
1863b77495aSJens Axboe
1873b77495aSJens Axboe if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
188bbbef3e9SMing Lei return;
1898435c6f3SJens Axboe
1906733e678SJens Axboe /* len == 0 is fine here, non-ring will always drop all of it */
1916733e678SJens Axboe __io_put_kbuf_list(req, 0, &req->ctx->io_buffers_comp);
1923b77495aSJens Axboe }
1933b77495aSJens Axboe
__io_put_kbufs(struct io_kiocb * req,int len,int nbufs,unsigned issue_flags)1946733e678SJens Axboe static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
1956733e678SJens Axboe int nbufs, unsigned issue_flags)
1963b77495aSJens Axboe {
1978435c6f3SJens Axboe unsigned int ret;
1983b77495aSJens Axboe
1998435c6f3SJens Axboe if (!(req->flags & (REQ_F_BUFFER_RING | REQ_F_BUFFER_SELECTED)))
2003b77495aSJens Axboe return 0;
2018435c6f3SJens Axboe
2028435c6f3SJens Axboe ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
203*ae98dbf4SJens Axboe if (req->flags & REQ_F_BUFFER_RING) {
204*ae98dbf4SJens Axboe if (!__io_put_kbuf_ring(req, len, nbufs))
205*ae98dbf4SJens Axboe ret |= IORING_CQE_F_BUF_MORE;
206*ae98dbf4SJens Axboe } else {
2076733e678SJens Axboe __io_put_kbuf(req, len, issue_flags);
208*ae98dbf4SJens Axboe }
2098435c6f3SJens Axboe return ret;
2103b77495aSJens Axboe }
21135c8711cSJens Axboe
io_put_kbuf(struct io_kiocb * req,int len,unsigned issue_flags)2126733e678SJens Axboe static inline unsigned int io_put_kbuf(struct io_kiocb *req, int len,
21335c8711cSJens Axboe unsigned issue_flags)
21435c8711cSJens Axboe {
2156733e678SJens Axboe return __io_put_kbufs(req, len, 1, issue_flags);
21635c8711cSJens Axboe }
21735c8711cSJens Axboe
io_put_kbufs(struct io_kiocb * req,int len,int nbufs,unsigned issue_flags)2186733e678SJens Axboe static inline unsigned int io_put_kbufs(struct io_kiocb *req, int len,
2196733e678SJens Axboe int nbufs, unsigned issue_flags)
22035c8711cSJens Axboe {
2216733e678SJens Axboe return __io_put_kbufs(req, len, nbufs, issue_flags);
22235c8711cSJens Axboe }
2233b77495aSJens Axboe #endif
224