13b77495aSJens Axboe // SPDX-License-Identifier: GPL-2.0 23b77495aSJens Axboe #ifndef IOU_KBUF_H 33b77495aSJens Axboe #define IOU_KBUF_H 43b77495aSJens Axboe 53b77495aSJens Axboe #include <uapi/linux/io_uring.h> 63b77495aSJens Axboe 73b77495aSJens Axboe struct io_buffer_list { 83b77495aSJens Axboe /* 93b77495aSJens Axboe * If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not, 103b77495aSJens Axboe * then these are classic provided buffers and ->buf_list is used. 113b77495aSJens Axboe */ 123b77495aSJens Axboe union { 133b77495aSJens Axboe struct list_head buf_list; 143b77495aSJens Axboe struct { 153b77495aSJens Axboe struct page **buf_pages; 163b77495aSJens Axboe struct io_uring_buf_ring *buf_ring; 173b77495aSJens Axboe }; 183b77495aSJens Axboe }; 193b77495aSJens Axboe __u16 bgid; 203b77495aSJens Axboe 213b77495aSJens Axboe /* below is for ring provided buffers */ 223b77495aSJens Axboe __u16 buf_nr_pages; 233b77495aSJens Axboe __u16 nr_entries; 243b77495aSJens Axboe __u16 head; 253b77495aSJens Axboe __u16 mask; 263b77495aSJens Axboe }; 273b77495aSJens Axboe 283b77495aSJens Axboe struct io_buffer { 293b77495aSJens Axboe struct list_head list; 303b77495aSJens Axboe __u64 addr; 313b77495aSJens Axboe __u32 len; 323b77495aSJens Axboe __u16 bid; 333b77495aSJens Axboe __u16 bgid; 343b77495aSJens Axboe }; 353b77495aSJens Axboe 363b77495aSJens Axboe void __user *io_buffer_select(struct io_kiocb *req, size_t *len, 373b77495aSJens Axboe unsigned int issue_flags); 383b77495aSJens Axboe void io_destroy_buffers(struct io_ring_ctx *ctx); 393b77495aSJens Axboe 403b77495aSJens Axboe int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 413b77495aSJens Axboe int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags); 423b77495aSJens Axboe 433b77495aSJens Axboe int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); 443b77495aSJens Axboe int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags); 453b77495aSJens Axboe 463b77495aSJens Axboe int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg); 473b77495aSJens Axboe int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg); 483b77495aSJens Axboe 4953ccf69bSPavel Begunkov unsigned int __io_put_kbuf(struct io_kiocb *req, unsigned issue_flags); 5053ccf69bSPavel Begunkov 51*024b8fdeSHao Xu void io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags); 52*024b8fdeSHao Xu void io_kbuf_recycle_ring(struct io_kiocb *req); 53*024b8fdeSHao Xu 543b77495aSJens Axboe static inline bool io_do_buffer_select(struct io_kiocb *req) 553b77495aSJens Axboe { 563b77495aSJens Axboe if (!(req->flags & REQ_F_BUFFER_SELECT)) 573b77495aSJens Axboe return false; 583b77495aSJens Axboe return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)); 593b77495aSJens Axboe } 603b77495aSJens Axboe 613b77495aSJens Axboe static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) 623b77495aSJens Axboe { 633b77495aSJens Axboe /* 643b77495aSJens Axboe * READV uses fields in `struct io_rw` (len/addr) to stash the selected 653b77495aSJens Axboe * buffer data. However if that buffer is recycled the original request 663b77495aSJens Axboe * data stored in addr is lost. Therefore forbid recycling for now. 673b77495aSJens Axboe */ 683b77495aSJens Axboe if (req->opcode == IORING_OP_READV) 693b77495aSJens Axboe return; 703b77495aSJens Axboe 71*024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_SELECTED) 72*024b8fdeSHao Xu io_kbuf_recycle_legacy(req, issue_flags); 73*024b8fdeSHao Xu if (req->flags & REQ_F_BUFFER_RING) 74*024b8fdeSHao Xu io_kbuf_recycle_ring(req); 753b77495aSJens Axboe } 763b77495aSJens Axboe 7753ccf69bSPavel Begunkov static inline unsigned int __io_put_kbuf_list(struct io_kiocb *req, 7853ccf69bSPavel Begunkov struct list_head *list) 793b77495aSJens Axboe { 803b77495aSJens Axboe if (req->flags & REQ_F_BUFFER_RING) { 813b77495aSJens Axboe if (req->buf_list) 823b77495aSJens Axboe req->buf_list->head++; 833b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_RING; 843b77495aSJens Axboe } else { 853b77495aSJens Axboe list_add(&req->kbuf->list, list); 863b77495aSJens Axboe req->flags &= ~REQ_F_BUFFER_SELECTED; 873b77495aSJens Axboe } 883b77495aSJens Axboe 893b77495aSJens Axboe return IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT); 903b77495aSJens Axboe } 913b77495aSJens Axboe 923b77495aSJens Axboe static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req) 933b77495aSJens Axboe { 943b77495aSJens Axboe lockdep_assert_held(&req->ctx->completion_lock); 953b77495aSJens Axboe 963b77495aSJens Axboe if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) 973b77495aSJens Axboe return 0; 9853ccf69bSPavel Begunkov return __io_put_kbuf_list(req, &req->ctx->io_buffers_comp); 993b77495aSJens Axboe } 1003b77495aSJens Axboe 1013b77495aSJens Axboe static inline unsigned int io_put_kbuf(struct io_kiocb *req, 1023b77495aSJens Axboe unsigned issue_flags) 1033b77495aSJens Axboe { 1043b77495aSJens Axboe 1053b77495aSJens Axboe if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) 1063b77495aSJens Axboe return 0; 10753ccf69bSPavel Begunkov return __io_put_kbuf(req, issue_flags); 1083b77495aSJens Axboe } 1093b77495aSJens Axboe #endif 110