1eb42cebbSPavel Begunkov #include <linux/kernel.h>
2eb42cebbSPavel Begunkov #include <linux/errno.h>
3eb42cebbSPavel Begunkov #include <linux/file.h>
4eb42cebbSPavel Begunkov #include <linux/slab.h>
5eb42cebbSPavel Begunkov #include <linux/net.h>
6eb42cebbSPavel Begunkov #include <linux/io_uring.h>
7eb42cebbSPavel Begunkov
8eb42cebbSPavel Begunkov #include "io_uring.h"
9eb42cebbSPavel Begunkov #include "notif.h"
1068ef5578SPavel Begunkov #include "rsrc.h"
11eb42cebbSPavel Begunkov
126fe42209SPavel Begunkov static const struct ubuf_info_ops io_ubuf_ops;
136fe42209SPavel Begunkov
io_notif_tw_complete(struct io_kiocb * notif,struct io_tw_state * ts)145a569469SPavel Begunkov static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
15eb42cebbSPavel Begunkov {
1614b146b6SPavel Begunkov struct io_notif_data *nd = io_notif_to_data(notif);
17eb42cebbSPavel Begunkov
186fe42209SPavel Begunkov do {
196fe42209SPavel Begunkov notif = cmd_to_io_kiocb(nd);
206fe42209SPavel Begunkov
216fe42209SPavel Begunkov lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0);
226fe42209SPavel Begunkov
236b7f864bSPavel Begunkov if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used))
2442385b02SPavel Begunkov notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED;
2542385b02SPavel Begunkov
262e730d8dSPavel Begunkov if (nd->account_pages && notif->ctx->user) {
272e730d8dSPavel Begunkov __io_unaccount_mem(notif->ctx->user, nd->account_pages);
2814b146b6SPavel Begunkov nd->account_pages = 0;
29e29e3bd4SPavel Begunkov }
306fe42209SPavel Begunkov
316fe42209SPavel Begunkov nd = nd->next;
32a282967cSPavel Begunkov io_req_task_complete(notif, ts);
336fe42209SPavel Begunkov } while (nd);
3440725d1bSPavel Begunkov }
3540725d1bSPavel Begunkov
io_tx_ubuf_complete(struct sk_buff * skb,struct ubuf_info * uarg,bool success)365a569469SPavel Begunkov void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
37eb42cebbSPavel Begunkov bool success)
38eb42cebbSPavel Begunkov {
3914b146b6SPavel Begunkov struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
4014b146b6SPavel Begunkov struct io_kiocb *notif = cmd_to_io_kiocb(nd);
41*19352a1dSPavel Begunkov unsigned tw_flags;
42eb42cebbSPavel Begunkov
43e307e669SStefan Metzmacher if (nd->zc_report) {
44e307e669SStefan Metzmacher if (success && !nd->zc_used && skb)
45e307e669SStefan Metzmacher WRITE_ONCE(nd->zc_used, true);
46e307e669SStefan Metzmacher else if (!success && !nd->zc_copied)
47e307e669SStefan Metzmacher WRITE_ONCE(nd->zc_copied, true);
48e307e669SStefan Metzmacher }
4999863292SPavel Begunkov
507e58d0afSPavel Begunkov if (!refcount_dec_and_test(&uarg->refcnt))
517e58d0afSPavel Begunkov return;
527e58d0afSPavel Begunkov
536fe42209SPavel Begunkov if (nd->head != nd) {
546fe42209SPavel Begunkov io_tx_ubuf_complete(skb, &nd->head->uarg, success);
556fe42209SPavel Begunkov return;
566fe42209SPavel Begunkov }
57*19352a1dSPavel Begunkov
58*19352a1dSPavel Begunkov tw_flags = nd->next ? 0 : IOU_F_TWQ_LAZY_WAKE;
596b7f864bSPavel Begunkov notif->io_task_work.func = io_notif_tw_complete;
60*19352a1dSPavel Begunkov __io_req_task_work_add(notif, tw_flags);
6140725d1bSPavel Begunkov }
62eb4a299bSPavel Begunkov
io_link_skb(struct sk_buff * skb,struct ubuf_info * uarg)636fe42209SPavel Begunkov static int io_link_skb(struct sk_buff *skb, struct ubuf_info *uarg)
646fe42209SPavel Begunkov {
656fe42209SPavel Begunkov struct io_notif_data *nd, *prev_nd;
666fe42209SPavel Begunkov struct io_kiocb *prev_notif, *notif;
676fe42209SPavel Begunkov struct ubuf_info *prev_uarg = skb_zcopy(skb);
686fe42209SPavel Begunkov
696fe42209SPavel Begunkov nd = container_of(uarg, struct io_notif_data, uarg);
706fe42209SPavel Begunkov notif = cmd_to_io_kiocb(nd);
716fe42209SPavel Begunkov
726fe42209SPavel Begunkov if (!prev_uarg) {
736fe42209SPavel Begunkov net_zcopy_get(&nd->uarg);
746fe42209SPavel Begunkov skb_zcopy_init(skb, &nd->uarg);
756fe42209SPavel Begunkov return 0;
766fe42209SPavel Begunkov }
776fe42209SPavel Begunkov /* handle it separately as we can't link a notif to itself */
786fe42209SPavel Begunkov if (unlikely(prev_uarg == &nd->uarg))
796fe42209SPavel Begunkov return 0;
806fe42209SPavel Begunkov /* we can't join two links together, just request a fresh skb */
816fe42209SPavel Begunkov if (unlikely(nd->head != nd || nd->next))
826fe42209SPavel Begunkov return -EEXIST;
836fe42209SPavel Begunkov /* don't mix zc providers */
846fe42209SPavel Begunkov if (unlikely(prev_uarg->ops != &io_ubuf_ops))
856fe42209SPavel Begunkov return -EEXIST;
866fe42209SPavel Begunkov
876fe42209SPavel Begunkov prev_nd = container_of(prev_uarg, struct io_notif_data, uarg);
886fe42209SPavel Begunkov prev_notif = cmd_to_io_kiocb(nd);
896fe42209SPavel Begunkov
906fe42209SPavel Begunkov /* make sure all noifications can be finished in the same task_work */
916fe42209SPavel Begunkov if (unlikely(notif->ctx != prev_notif->ctx ||
926fe42209SPavel Begunkov notif->task != prev_notif->task))
936fe42209SPavel Begunkov return -EEXIST;
946fe42209SPavel Begunkov
956fe42209SPavel Begunkov nd->head = prev_nd->head;
966fe42209SPavel Begunkov nd->next = prev_nd->next;
976fe42209SPavel Begunkov prev_nd->next = nd;
986fe42209SPavel Begunkov net_zcopy_get(&nd->head->uarg);
996fe42209SPavel Begunkov return 0;
1006fe42209SPavel Begunkov }
1016fe42209SPavel Begunkov
1027ab4f16fSPavel Begunkov static const struct ubuf_info_ops io_ubuf_ops = {
1037ab4f16fSPavel Begunkov .complete = io_tx_ubuf_complete,
1046fe42209SPavel Begunkov .link_skb = io_link_skb,
1057ab4f16fSPavel Begunkov };
1067ab4f16fSPavel Begunkov
io_alloc_notif(struct io_ring_ctx * ctx)107b48c312bSPavel Begunkov struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
108eb42cebbSPavel Begunkov __must_hold(&ctx->uring_lock)
109eb42cebbSPavel Begunkov {
11014b146b6SPavel Begunkov struct io_kiocb *notif;
11114b146b6SPavel Begunkov struct io_notif_data *nd;
112eb42cebbSPavel Begunkov
113c8576f3eSPavel Begunkov if (unlikely(!io_alloc_req(ctx, ¬if)))
114eb42cebbSPavel Begunkov return NULL;
11514b146b6SPavel Begunkov notif->opcode = IORING_OP_NOP;
11614b146b6SPavel Begunkov notif->flags = 0;
11714b146b6SPavel Begunkov notif->file = NULL;
11814b146b6SPavel Begunkov notif->task = current;
11914b146b6SPavel Begunkov io_get_task_refs(1);
12014b146b6SPavel Begunkov notif->rsrc_node = NULL;
121eb4a299bSPavel Begunkov
12214b146b6SPavel Begunkov nd = io_notif_to_data(notif);
12399863292SPavel Begunkov nd->zc_report = false;
12499863292SPavel Begunkov nd->account_pages = 0;
1256fe42209SPavel Begunkov nd->next = NULL;
1266fe42209SPavel Begunkov nd->head = nd;
1276fe42209SPavel Begunkov
128519760dfSPavel Begunkov nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
1297ab4f16fSPavel Begunkov nd->uarg.ops = &io_ubuf_ops;
13014b146b6SPavel Begunkov refcount_set(&nd->uarg.refcnt, 1);
131eb42cebbSPavel Begunkov return notif;
132eb42cebbSPavel Begunkov }
133