1 #include <linux/kernel.h> 2 #include <linux/errno.h> 3 #include <linux/file.h> 4 #include <linux/slab.h> 5 #include <linux/net.h> 6 #include <linux/io_uring.h> 7 8 #include "io_uring.h" 9 #include "notif.h" 10 #include "rsrc.h" 11 12 static void io_notif_complete_tw_ext(struct io_kiocb *notif, bool *locked) 13 { 14 struct io_notif_data *nd = io_notif_to_data(notif); 15 struct io_ring_ctx *ctx = notif->ctx; 16 17 if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) 18 notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED; 19 20 if (nd->account_pages && ctx->user) { 21 __io_unaccount_mem(ctx->user, nd->account_pages); 22 nd->account_pages = 0; 23 } 24 io_req_task_complete(notif, locked); 25 } 26 27 static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg, 28 bool success) 29 { 30 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); 31 struct io_kiocb *notif = cmd_to_io_kiocb(nd); 32 33 if (refcount_dec_and_test(&uarg->refcnt)) 34 io_req_task_work_add(notif); 35 } 36 37 static void io_tx_ubuf_callback_ext(struct sk_buff *skb, struct ubuf_info *uarg, 38 bool success) 39 { 40 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); 41 42 if (nd->zc_report) { 43 if (success && !nd->zc_used && skb) 44 WRITE_ONCE(nd->zc_used, true); 45 else if (!success && !nd->zc_copied) 46 WRITE_ONCE(nd->zc_copied, true); 47 } 48 io_tx_ubuf_callback(skb, uarg, success); 49 } 50 51 void io_notif_set_extended(struct io_kiocb *notif) 52 { 53 struct io_notif_data *nd = io_notif_to_data(notif); 54 55 if (nd->uarg.callback != io_tx_ubuf_callback_ext) { 56 nd->account_pages = 0; 57 nd->zc_report = false; 58 nd->zc_used = false; 59 nd->zc_copied = false; 60 nd->uarg.callback = io_tx_ubuf_callback_ext; 61 notif->io_task_work.func = io_notif_complete_tw_ext; 62 } 63 } 64 65 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) 66 __must_hold(&ctx->uring_lock) 67 { 68 struct io_kiocb *notif; 69 struct io_notif_data *nd; 70 71 if (unlikely(!io_alloc_req(ctx, ¬if))) 72 return NULL; 73 notif->opcode = IORING_OP_NOP; 74 notif->flags = 0; 75 notif->file = NULL; 76 notif->task = current; 77 io_get_task_refs(1); 78 notif->rsrc_node = NULL; 79 notif->io_task_work.func = io_req_task_complete; 80 81 nd = io_notif_to_data(notif); 82 nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 83 nd->uarg.callback = io_tx_ubuf_callback; 84 refcount_set(&nd->uarg.refcnt, 1); 85 return notif; 86 } 87