1 #include <linux/kernel.h> 2 #include <linux/errno.h> 3 #include <linux/file.h> 4 #include <linux/slab.h> 5 #include <linux/net.h> 6 #include <linux/io_uring.h> 7 8 #include "io_uring.h" 9 #include "notif.h" 10 #include "rsrc.h" 11 12 static void io_notif_complete_tw_ext(struct io_kiocb *notif, struct io_tw_state *ts) 13 { 14 struct io_notif_data *nd = io_notif_to_data(notif); 15 struct io_ring_ctx *ctx = notif->ctx; 16 17 if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) 18 notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED; 19 20 if (nd->account_pages && ctx->user) { 21 __io_unaccount_mem(ctx->user, nd->account_pages); 22 nd->account_pages = 0; 23 } 24 io_req_task_complete(notif, ts); 25 } 26 27 static void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg, 28 bool success) 29 { 30 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); 31 struct io_kiocb *notif = cmd_to_io_kiocb(nd); 32 33 if (refcount_dec_and_test(&uarg->refcnt)) 34 __io_req_task_work_add(notif, IOU_F_TWQ_LAZY_WAKE); 35 } 36 37 static void io_tx_ubuf_callback_ext(struct sk_buff *skb, struct ubuf_info *uarg, 38 bool success) 39 { 40 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); 41 42 if (nd->zc_report) { 43 if (success && !nd->zc_used && skb) 44 WRITE_ONCE(nd->zc_used, true); 45 else if (!success && !nd->zc_copied) 46 WRITE_ONCE(nd->zc_copied, true); 47 } 48 io_tx_ubuf_complete(skb, uarg, success); 49 } 50 51 static const struct ubuf_info_ops io_ubuf_ops = { 52 .complete = io_tx_ubuf_complete, 53 }; 54 55 static const struct ubuf_info_ops io_ubuf_ops_ext = { 56 .complete = io_tx_ubuf_callback_ext, 57 }; 58 59 void io_notif_set_extended(struct io_kiocb *notif) 60 { 61 struct io_notif_data *nd = io_notif_to_data(notif); 62 63 if (nd->uarg.ops != &io_ubuf_ops_ext) { 64 nd->account_pages = 0; 65 nd->zc_report = false; 66 nd->zc_used = false; 67 nd->zc_copied = false; 68 nd->uarg.ops = &io_ubuf_ops_ext; 69 notif->io_task_work.func = io_notif_complete_tw_ext; 70 } 71 } 72 73 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) 74 __must_hold(&ctx->uring_lock) 75 { 76 struct io_kiocb *notif; 77 struct io_notif_data *nd; 78 79 if (unlikely(!io_alloc_req(ctx, ¬if))) 80 return NULL; 81 notif->opcode = IORING_OP_NOP; 82 notif->flags = 0; 83 notif->file = NULL; 84 notif->task = current; 85 io_get_task_refs(1); 86 notif->rsrc_node = NULL; 87 notif->io_task_work.func = io_req_task_complete; 88 89 nd = io_notif_to_data(notif); 90 nd->uarg.flags = IO_NOTIF_UBUF_FLAGS; 91 nd->uarg.ops = &io_ubuf_ops; 92 refcount_set(&nd->uarg.refcnt, 1); 93 return notif; 94 } 95