1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/slab.h> 6 #include <linux/net.h> 7 #include <linux/io_uring.h> 8 9 #include "io_uring.h" 10 #include "notif.h" 11 #include "rsrc.h" 12 13 static const struct ubuf_info_ops io_ubuf_ops; 14 15 static void io_notif_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw) 16 { 17 struct io_kiocb *notif = tw_req.req; 18 struct io_notif_data *nd = io_notif_to_data(notif); 19 struct io_ring_ctx *ctx = notif->ctx; 20 21 lockdep_assert_held(&ctx->uring_lock); 22 23 do { 24 notif = cmd_to_io_kiocb(nd); 25 26 if (WARN_ON_ONCE(ctx != notif->ctx)) 27 return; 28 lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0); 29 30 if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used)) 31 notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED; 32 33 if (nd->account_pages && notif->ctx->user) { 34 __io_unaccount_mem(notif->ctx->user, nd->account_pages); 35 nd->account_pages = 0; 36 } 37 38 nd = nd->next; 39 io_req_task_complete((struct io_tw_req){notif}, tw); 40 } while (nd); 41 } 42 43 void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg, 44 bool success) 45 { 46 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); 47 struct io_kiocb *notif = cmd_to_io_kiocb(nd); 48 unsigned tw_flags; 49 50 if (nd->zc_report) { 51 if (success && !nd->zc_used && skb) 52 WRITE_ONCE(nd->zc_used, true); 53 else if (!success && !nd->zc_copied) 54 WRITE_ONCE(nd->zc_copied, true); 55 } 56 57 if (!refcount_dec_and_test(&uarg->refcnt)) 58 return; 59 60 if (nd->head != nd) { 61 io_tx_ubuf_complete(skb, &nd->head->uarg, success); 62 return; 63 } 64 65 tw_flags = nd->next ? 0 : IOU_F_TWQ_LAZY_WAKE; 66 notif->io_task_work.func = io_notif_tw_complete; 67 __io_req_task_work_add(notif, tw_flags); 68 } 69 70 static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg) 71 { 72 struct io_notif_data *nd, *prev_nd; 73 struct io_kiocb *prev_notif, *notif; 74 struct ubuf_info *prev_uarg = skb_zcopy(skb); 75 76 nd = container_of(uarg, struct io_notif_data, uarg); 77 notif = cmd_to_io_kiocb(nd); 78 79 if (!prev_uarg) { 80 net_zcopy_get(&nd->uarg); 81 skb_zcopy_init(skb, &nd->uarg); 82 return 0; 83 } 84 /* handle it separately as we can't link a notif to itself */ 85 if (unlikely(prev_uarg == &nd->uarg)) 86 return 0; 87 /* we can't join two links together, just request a fresh skb */ 88 if (unlikely(nd->head != nd || nd->next)) 89 return -EEXIST; 90 /* don't mix zc providers */ 91 if (unlikely(prev_uarg->ops != &io_ubuf_ops)) 92 return -EEXIST; 93 94 prev_nd = container_of(prev_uarg, struct io_notif_data, uarg); 95 prev_notif = cmd_to_io_kiocb(prev_nd); 96 97 /* make sure all notifications can be finished in the same task_work */ 98 if (unlikely(notif->ctx != prev_notif->ctx || 99 notif->tctx != prev_notif->tctx)) 100 return -EEXIST; 101 102 nd->head = prev_nd->head; 103 nd->next = prev_nd->next; 104 prev_nd->next = nd; 105 net_zcopy_get(&nd->head->uarg); 106 return 0; 107 } 108 109 static const struct ubuf_info_ops io_ubuf_ops = { 110 .complete = io_tx_ubuf_complete, 111 .link_skb = io_link_skb, 112 }; 113 114 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) 115 __must_hold(&ctx->uring_lock) 116 { 117 struct io_kiocb *notif; 118 struct io_notif_data *nd; 119 120 if (unlikely(!io_alloc_req(ctx, ¬if))) 121 return NULL; 122 notif->ctx = ctx; 123 notif->opcode = IORING_OP_NOP; 124 notif->flags = 0; 125 notif->file = NULL; 126 notif->tctx = current->io_uring; 127 io_get_task_refs(1); 128 notif->file_node = NULL; 129 notif->buf_node = NULL; 130 131 nd = io_notif_to_data(notif); 132 nd->zc_report = false; 133 nd->account_pages = 0; 134 nd->next = NULL; 135 nd->head = nd; 136 137 nd->uarg.flags = IO_NOTIF_UBUF_FLAGS; 138 nd->uarg.ops = &io_ubuf_ops; 139 refcount_set(&nd->uarg.refcnt, 1); 140 return notif; 141 } 142