1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/slab.h>
6 #include <linux/net.h>
7 #include <linux/un.h>
8 #include <linux/compat.h>
9 #include <net/compat.h>
10 #include <linux/io_uring.h>
11
12 #include <uapi/linux/io_uring.h>
13
14 #include "filetable.h"
15 #include "io_uring.h"
16 #include "kbuf.h"
17 #include "alloc_cache.h"
18 #include "net.h"
19 #include "notif.h"
20 #include "rsrc.h"
21 #include "zcrx.h"
22
23 struct io_shutdown {
24 struct file *file;
25 int how;
26 };
27
28 struct io_accept {
29 struct file *file;
30 struct sockaddr __user *addr;
31 int __user *addr_len;
32 int flags;
33 int iou_flags;
34 u32 file_slot;
35 unsigned long nofile;
36 };
37
38 struct io_socket {
39 struct file *file;
40 int domain;
41 int type;
42 int protocol;
43 int flags;
44 u32 file_slot;
45 unsigned long nofile;
46 };
47
48 struct io_connect {
49 struct file *file;
50 struct sockaddr __user *addr;
51 int addr_len;
52 bool in_progress;
53 bool seen_econnaborted;
54 };
55
56 struct io_bind {
57 struct file *file;
58 int addr_len;
59 };
60
61 struct io_listen {
62 struct file *file;
63 int backlog;
64 };
65
66 struct io_sr_msg {
67 struct file *file;
68 union {
69 struct compat_msghdr __user *umsg_compat;
70 struct user_msghdr __user *umsg;
71 void __user *buf;
72 };
73 int len;
74 unsigned done_io;
75 unsigned msg_flags;
76 unsigned nr_multishot_loops;
77 u16 flags;
78 /* initialised and used only by !msg send variants */
79 u16 buf_group;
80 /* per-invocation mshot limit */
81 unsigned mshot_len;
82 /* overall mshot byte limit */
83 unsigned mshot_total_len;
84 void __user *msg_control;
85 /* used only for send zerocopy */
86 struct io_kiocb *notif;
87 };
88
89 /*
90 * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold
91 * anyway. Use the upper 8 bits for internal uses.
92 */
93 enum sr_retry_flags {
94 IORING_RECV_RETRY = (1U << 15),
95 IORING_RECV_PARTIAL_MAP = (1U << 14),
96 IORING_RECV_MSHOT_CAP = (1U << 13),
97 IORING_RECV_MSHOT_LIM = (1U << 12),
98 IORING_RECV_MSHOT_DONE = (1U << 11),
99
100 IORING_RECV_RETRY_CLEAR = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP,
101 IORING_RECV_NO_RETRY = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP |
102 IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE,
103 };
104
105 /*
106 * Number of times we'll try and do receives if there's more data. If we
107 * exceed this limit, then add us to the back of the queue and retry from
108 * there. This helps fairness between flooding clients.
109 */
110 #define MULTISHOT_MAX_RETRY 32
111
112 struct io_recvzc {
113 struct file *file;
114 u16 flags;
115 u32 len;
116 struct io_zcrx_ifq *ifq;
117 };
118
119 static int io_sg_from_iter_iovec(struct sk_buff *skb,
120 struct iov_iter *from, size_t length);
121 static int io_sg_from_iter(struct sk_buff *skb,
122 struct iov_iter *from, size_t length);
123
io_shutdown_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)124 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
125 {
126 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
127
128 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
129 sqe->buf_index || sqe->splice_fd_in))
130 return -EINVAL;
131
132 shutdown->how = READ_ONCE(sqe->len);
133 req->flags |= REQ_F_FORCE_ASYNC;
134 return 0;
135 }
136
io_shutdown(struct io_kiocb * req,unsigned int issue_flags)137 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
138 {
139 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
140 struct socket *sock;
141 int ret;
142
143 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
144
145 sock = sock_from_file(req->file);
146 if (unlikely(!sock))
147 return -ENOTSOCK;
148
149 ret = __sys_shutdown_sock(sock, shutdown->how);
150 io_req_set_res(req, ret, 0);
151 return IOU_COMPLETE;
152 }
153
io_net_retry(struct socket * sock,int flags)154 static bool io_net_retry(struct socket *sock, int flags)
155 {
156 if (!(flags & MSG_WAITALL))
157 return false;
158 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
159 }
160
io_netmsg_iovec_free(struct io_async_msghdr * kmsg)161 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)
162 {
163 if (kmsg->vec.iovec)
164 io_vec_free(&kmsg->vec);
165 }
166
io_netmsg_recycle(struct io_kiocb * req,unsigned int issue_flags)167 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
168 {
169 struct io_async_msghdr *hdr = req->async_data;
170
171 /* can't recycle, ensure we free the iovec if we have one */
172 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
173 io_netmsg_iovec_free(hdr);
174 return;
175 }
176
177 /* Let normal cleanup path reap it if we fail adding to the cache */
178 io_alloc_cache_vec_kasan(&hdr->vec);
179 if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP)
180 io_vec_free(&hdr->vec);
181
182 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr))
183 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
184 }
185
io_msg_alloc_async(struct io_kiocb * req)186 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
187 {
188 struct io_ring_ctx *ctx = req->ctx;
189 struct io_async_msghdr *hdr;
190
191 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req);
192 if (!hdr)
193 return NULL;
194
195 /* If the async data was cached, we might have an iov cached inside. */
196 if (hdr->vec.iovec)
197 req->flags |= REQ_F_NEED_CLEANUP;
198 return hdr;
199 }
200
io_mshot_prep_retry(struct io_kiocb * req,struct io_async_msghdr * kmsg)201 static inline void io_mshot_prep_retry(struct io_kiocb *req,
202 struct io_async_msghdr *kmsg)
203 {
204 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
205
206 req->flags &= ~REQ_F_BL_EMPTY;
207 sr->done_io = 0;
208 sr->flags &= ~IORING_RECV_RETRY_CLEAR;
209 sr->len = sr->mshot_len;
210 }
211
io_net_import_vec(struct io_kiocb * req,struct io_async_msghdr * iomsg,const struct iovec __user * uiov,unsigned uvec_seg,int ddir)212 static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg,
213 const struct iovec __user *uiov, unsigned uvec_seg,
214 int ddir)
215 {
216 struct iovec *iov;
217 int ret, nr_segs;
218
219 if (iomsg->vec.iovec) {
220 nr_segs = iomsg->vec.nr;
221 iov = iomsg->vec.iovec;
222 } else {
223 nr_segs = 1;
224 iov = &iomsg->fast_iov;
225 }
226
227 ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov,
228 &iomsg->msg.msg_iter, io_is_compat(req->ctx));
229 if (unlikely(ret < 0))
230 return ret;
231
232 if (iov) {
233 req->flags |= REQ_F_NEED_CLEANUP;
234 io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs);
235 }
236 return 0;
237 }
238
io_copy_msghdr_from_user(struct user_msghdr * msg,struct user_msghdr __user * umsg)239 static int io_copy_msghdr_from_user(struct user_msghdr *msg,
240 struct user_msghdr __user *umsg)
241 {
242 if (!user_access_begin(umsg, sizeof(*umsg)))
243 return -EFAULT;
244 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end);
245 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end);
246 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end);
247 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end);
248 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end);
249 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end);
250 user_access_end();
251 return 0;
252 ua_end:
253 user_access_end();
254 return -EFAULT;
255 }
256
io_msg_copy_hdr(struct io_kiocb * req,struct io_async_msghdr * iomsg,struct user_msghdr * msg,struct sockaddr __user ** save_addr)257 static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
258 struct user_msghdr *msg,
259 struct sockaddr __user **save_addr)
260 {
261 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
262 int ret;
263
264 iomsg->msg.msg_name = &iomsg->addr;
265 iomsg->msg.msg_iter.nr_segs = 0;
266
267 if (io_is_compat(req->ctx)) {
268 struct compat_msghdr cmsg;
269
270 if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg)))
271 return -EFAULT;
272
273 ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr);
274 if (ret)
275 return ret;
276
277 memset(msg, 0, sizeof(*msg));
278 msg->msg_namelen = cmsg.msg_namelen;
279 msg->msg_controllen = cmsg.msg_controllen;
280 msg->msg_iov = compat_ptr(cmsg.msg_iov);
281 msg->msg_iovlen = cmsg.msg_iovlen;
282 } else {
283 ret = io_copy_msghdr_from_user(msg, sr->umsg);
284 if (unlikely(ret))
285 return ret;
286
287 msg->msg_flags = 0;
288
289 ret = __copy_msghdr(&iomsg->msg, msg, save_addr);
290 if (ret)
291 return ret;
292 }
293
294 if (req->flags & REQ_F_BUFFER_SELECT) {
295 if (msg->msg_iovlen == 0) {
296 sr->len = 0;
297 } else if (msg->msg_iovlen > 1) {
298 return -EINVAL;
299 } else {
300 struct iovec fast_iov, *iov;
301
302 iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov,
303 io_is_compat(req->ctx));
304 if (IS_ERR(iov))
305 return PTR_ERR(iov);
306 sr->len = iov->iov_len;
307 }
308 }
309 return 0;
310 }
311
io_sendmsg_recvmsg_cleanup(struct io_kiocb * req)312 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
313 {
314 struct io_async_msghdr *io = req->async_data;
315
316 io_netmsg_iovec_free(io);
317 }
318
io_send_setup(struct io_kiocb * req,const struct io_uring_sqe * sqe)319 static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
320 {
321 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
322 struct io_async_msghdr *kmsg = req->async_data;
323 void __user *addr;
324 u16 addr_len;
325 int ret;
326
327 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
328
329 if (READ_ONCE(sqe->__pad3[0]))
330 return -EINVAL;
331
332 kmsg->msg.msg_name = NULL;
333 kmsg->msg.msg_namelen = 0;
334 kmsg->msg.msg_control = NULL;
335 kmsg->msg.msg_controllen = 0;
336 kmsg->msg.msg_ubuf = NULL;
337
338 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
339 addr_len = READ_ONCE(sqe->addr_len);
340 if (addr) {
341 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);
342 if (unlikely(ret < 0))
343 return ret;
344 kmsg->msg.msg_name = &kmsg->addr;
345 kmsg->msg.msg_namelen = addr_len;
346 }
347 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
348 if (!(sr->flags & IORING_SEND_VECTORIZED)) {
349 req->flags |= REQ_F_IMPORT_BUFFER;
350 return 0;
351 }
352
353 kmsg->msg.msg_iter.nr_segs = sr->len;
354 return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len);
355 }
356 if (req->flags & REQ_F_BUFFER_SELECT)
357 return 0;
358
359 if (sr->flags & IORING_SEND_VECTORIZED)
360 return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE);
361
362 return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);
363 }
364
io_sendmsg_setup(struct io_kiocb * req,const struct io_uring_sqe * sqe)365 static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
366 {
367 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
368 struct io_async_msghdr *kmsg = req->async_data;
369 struct user_msghdr msg;
370 int ret;
371
372 sr->flags |= IORING_SEND_VECTORIZED;
373 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
374 ret = io_msg_copy_hdr(req, kmsg, &msg, NULL);
375 if (unlikely(ret))
376 return ret;
377 /* save msg_control as sys_sendmsg() overwrites it */
378 sr->msg_control = kmsg->msg.msg_control_user;
379
380 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
381 kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen;
382 return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov,
383 msg.msg_iovlen);
384 }
385 if (req->flags & REQ_F_BUFFER_SELECT)
386 return 0;
387 return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE);
388 }
389
390 #define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | \
391 IORING_SEND_VECTORIZED | IORING_RECVSEND_FIXED_BUF)
392
io_sendmsg_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)393 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
394 {
395 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
396
397 sr->done_io = 0;
398 sr->len = READ_ONCE(sqe->len);
399 if (unlikely(sr->len < 0))
400 return -EINVAL;
401 sr->flags = READ_ONCE(sqe->ioprio);
402 if (sr->flags & ~SENDMSG_FLAGS)
403 return -EINVAL;
404 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
405 /* registered buffer send only supported for plain IORING_OP_SEND */
406 if (req->opcode != IORING_OP_SEND ||
407 (req->flags & REQ_F_BUFFER_SELECT) ||
408 (sr->flags & (IORING_RECVSEND_BUNDLE|IORING_SEND_VECTORIZED)))
409 return -EINVAL;
410 req->buf_index = READ_ONCE(sqe->buf_index);
411 }
412 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
413 if (sr->msg_flags & MSG_DONTWAIT)
414 req->flags |= REQ_F_NOWAIT;
415 if (req->flags & REQ_F_BUFFER_SELECT)
416 sr->buf_group = req->buf_index;
417 sr->mshot_total_len = sr->mshot_len = 0;
418 if (sr->flags & IORING_RECVSEND_BUNDLE) {
419 if (req->opcode == IORING_OP_SENDMSG)
420 return -EINVAL;
421 sr->msg_flags |= MSG_WAITALL;
422 req->flags |= REQ_F_MULTISHOT;
423 }
424
425 if (io_is_compat(req->ctx))
426 sr->msg_flags |= MSG_CMSG_COMPAT;
427
428 if (unlikely(!io_msg_alloc_async(req)))
429 return -ENOMEM;
430 if (req->opcode != IORING_OP_SENDMSG)
431 return io_send_setup(req, sqe);
432 if (unlikely(sqe->addr2 || sqe->file_index))
433 return -EINVAL;
434 return io_sendmsg_setup(req, sqe);
435 }
436
io_req_msg_cleanup(struct io_kiocb * req,unsigned int issue_flags)437 static void io_req_msg_cleanup(struct io_kiocb *req,
438 unsigned int issue_flags)
439 {
440 io_netmsg_recycle(req, issue_flags);
441 }
442
443 /*
444 * For bundle completions, we need to figure out how many segments we consumed.
445 * A bundle could be using a single ITER_UBUF if that's all we mapped, or it
446 * could be using an ITER_IOVEC. If the latter, then if we consumed all of
447 * the segments, then it's a trivial questiont o answer. If we have residual
448 * data in the iter, then loop the segments to figure out how much we
449 * transferred.
450 */
io_bundle_nbufs(struct io_async_msghdr * kmsg,int ret)451 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)
452 {
453 struct iovec *iov;
454 int nbufs;
455
456 /* no data is always zero segments, and a ubuf is always 1 segment */
457 if (ret <= 0)
458 return 0;
459 if (iter_is_ubuf(&kmsg->msg.msg_iter))
460 return 1;
461
462 iov = kmsg->vec.iovec;
463 if (!iov)
464 iov = &kmsg->fast_iov;
465
466 /* if all data was transferred, it's basic pointer math */
467 if (!iov_iter_count(&kmsg->msg.msg_iter))
468 return iter_iov(&kmsg->msg.msg_iter) - iov;
469
470 /* short transfer, count segments */
471 nbufs = 0;
472 do {
473 int this_len = min_t(int, iov[nbufs].iov_len, ret);
474
475 nbufs++;
476 ret -= this_len;
477 } while (ret);
478
479 return nbufs;
480 }
481
io_net_kbuf_recyle(struct io_kiocb * req,struct io_buffer_list * bl,struct io_async_msghdr * kmsg,int len)482 static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,
483 struct io_async_msghdr *kmsg, int len)
484 {
485 req->flags |= REQ_F_BL_NO_RECYCLE;
486 if (req->flags & REQ_F_BUFFERS_COMMIT)
487 io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));
488 return IOU_RETRY;
489 }
490
io_send_finish(struct io_kiocb * req,struct io_async_msghdr * kmsg,struct io_br_sel * sel)491 static inline bool io_send_finish(struct io_kiocb *req,
492 struct io_async_msghdr *kmsg,
493 struct io_br_sel *sel)
494 {
495 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
496 bool bundle_finished = sel->val <= 0;
497 unsigned int cflags;
498
499 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {
500 cflags = io_put_kbuf(req, sel->val, sel->buf_list);
501 goto finish;
502 }
503
504 cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val));
505
506 /*
507 * Don't start new bundles if the buffer list is empty, or if the
508 * current operation needed to go through polling to complete.
509 */
510 if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED))
511 goto finish;
512
513 /*
514 * Fill CQE for this receive and see if we should keep trying to
515 * receive from this socket.
516 */
517 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
518 io_mshot_prep_retry(req, kmsg);
519 return false;
520 }
521
522 /* Otherwise stop bundle and use the current result. */
523 finish:
524 io_req_set_res(req, sel->val, cflags);
525 sel->val = IOU_COMPLETE;
526 return true;
527 }
528
io_sendmsg(struct io_kiocb * req,unsigned int issue_flags)529 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
530 {
531 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
532 struct io_async_msghdr *kmsg = req->async_data;
533 struct socket *sock;
534 unsigned flags;
535 int min_ret = 0;
536 int ret;
537
538 sock = sock_from_file(req->file);
539 if (unlikely(!sock))
540 return -ENOTSOCK;
541
542 if (!(req->flags & REQ_F_POLLED) &&
543 (sr->flags & IORING_RECVSEND_POLL_FIRST))
544 return -EAGAIN;
545
546 flags = sr->msg_flags;
547 if (issue_flags & IO_URING_F_NONBLOCK)
548 flags |= MSG_DONTWAIT;
549 if (flags & MSG_WAITALL)
550 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
551
552 kmsg->msg.msg_control_user = sr->msg_control;
553
554 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
555
556 if (ret < min_ret) {
557 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
558 return -EAGAIN;
559 if (ret > 0 && io_net_retry(sock, flags)) {
560 kmsg->msg.msg_controllen = 0;
561 kmsg->msg.msg_control = NULL;
562 sr->done_io += ret;
563 return -EAGAIN;
564 }
565 if (ret == -ERESTARTSYS)
566 ret = -EINTR;
567 req_set_fail(req);
568 }
569 io_req_msg_cleanup(req, issue_flags);
570 if (ret >= 0)
571 ret += sr->done_io;
572 else if (sr->done_io)
573 ret = sr->done_io;
574 io_req_set_res(req, ret, 0);
575 return IOU_COMPLETE;
576 }
577
io_send_select_buffer(struct io_kiocb * req,unsigned int issue_flags,struct io_br_sel * sel,struct io_async_msghdr * kmsg)578 static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
579 struct io_br_sel *sel, struct io_async_msghdr *kmsg)
580 {
581 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
582 struct buf_sel_arg arg = {
583 .iovs = &kmsg->fast_iov,
584 .max_len = min_not_zero(sr->len, INT_MAX),
585 .nr_iovs = 1,
586 .buf_group = sr->buf_group,
587 };
588 int ret;
589
590 if (kmsg->vec.iovec) {
591 arg.nr_iovs = kmsg->vec.nr;
592 arg.iovs = kmsg->vec.iovec;
593 arg.mode = KBUF_MODE_FREE;
594 }
595
596 if (!(sr->flags & IORING_RECVSEND_BUNDLE))
597 arg.nr_iovs = 1;
598 else
599 arg.mode |= KBUF_MODE_EXPAND;
600
601 ret = io_buffers_select(req, &arg, sel, issue_flags);
602 if (unlikely(ret < 0))
603 return ret;
604
605 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
606 kmsg->vec.nr = ret;
607 kmsg->vec.iovec = arg.iovs;
608 req->flags |= REQ_F_NEED_CLEANUP;
609 }
610 sr->len = arg.out_len;
611
612 if (ret == 1) {
613 sr->buf = arg.iovs[0].iov_base;
614 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,
615 &kmsg->msg.msg_iter);
616 if (unlikely(ret))
617 return ret;
618 } else {
619 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,
620 arg.iovs, ret, arg.out_len);
621 }
622
623 return 0;
624 }
625
io_send(struct io_kiocb * req,unsigned int issue_flags)626 int io_send(struct io_kiocb *req, unsigned int issue_flags)
627 {
628 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
629 struct io_async_msghdr *kmsg = req->async_data;
630 struct io_br_sel sel = { };
631 struct socket *sock;
632 unsigned flags;
633 int min_ret = 0;
634 int ret;
635
636 sock = sock_from_file(req->file);
637 if (unlikely(!sock))
638 return -ENOTSOCK;
639
640 if (!(req->flags & REQ_F_POLLED) &&
641 (sr->flags & IORING_RECVSEND_POLL_FIRST))
642 return -EAGAIN;
643
644 if (req->flags & REQ_F_IMPORT_BUFFER) {
645 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter,
646 (u64)(uintptr_t)sr->buf, sr->len,
647 ITER_SOURCE, issue_flags);
648 if (unlikely(ret))
649 return ret;
650 req->flags &= ~REQ_F_IMPORT_BUFFER;
651 }
652
653 flags = sr->msg_flags;
654 if (issue_flags & IO_URING_F_NONBLOCK)
655 flags |= MSG_DONTWAIT;
656
657 retry_bundle:
658 sel.buf_list = NULL;
659 if (io_do_buffer_select(req)) {
660 ret = io_send_select_buffer(req, issue_flags, &sel, kmsg);
661 if (ret)
662 return ret;
663 }
664
665 /*
666 * If MSG_WAITALL is set, or this is a bundle send, then we need
667 * the full amount. If just bundle is set, if we do a short send
668 * then we complete the bundle sequence rather than continue on.
669 */
670 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE)
671 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
672
673 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
674 kmsg->msg.msg_flags = flags;
675 ret = sock_sendmsg(sock, &kmsg->msg);
676 if (ret < min_ret) {
677 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
678 return -EAGAIN;
679
680 if (ret > 0 && io_net_retry(sock, flags)) {
681 sr->len -= ret;
682 sr->buf += ret;
683 sr->done_io += ret;
684 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
685 }
686 if (ret == -ERESTARTSYS)
687 ret = -EINTR;
688 req_set_fail(req);
689 }
690 if (ret >= 0)
691 ret += sr->done_io;
692 else if (sr->done_io)
693 ret = sr->done_io;
694
695 sel.val = ret;
696 if (!io_send_finish(req, kmsg, &sel))
697 goto retry_bundle;
698
699 io_req_msg_cleanup(req, issue_flags);
700 return sel.val;
701 }
702
io_recvmsg_mshot_prep(struct io_kiocb * req,struct io_async_msghdr * iomsg,int namelen,size_t controllen)703 static int io_recvmsg_mshot_prep(struct io_kiocb *req,
704 struct io_async_msghdr *iomsg,
705 int namelen, size_t controllen)
706 {
707 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) ==
708 (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) {
709 int hdr;
710
711 if (unlikely(namelen < 0))
712 return -EOVERFLOW;
713 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out),
714 namelen, &hdr))
715 return -EOVERFLOW;
716 if (check_add_overflow(hdr, controllen, &hdr))
717 return -EOVERFLOW;
718
719 iomsg->namelen = namelen;
720 iomsg->controllen = controllen;
721 return 0;
722 }
723
724 return 0;
725 }
726
io_recvmsg_copy_hdr(struct io_kiocb * req,struct io_async_msghdr * iomsg)727 static int io_recvmsg_copy_hdr(struct io_kiocb *req,
728 struct io_async_msghdr *iomsg)
729 {
730 struct user_msghdr msg;
731 int ret;
732
733 ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr);
734 if (unlikely(ret))
735 return ret;
736
737 if (!(req->flags & REQ_F_BUFFER_SELECT)) {
738 ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen,
739 ITER_DEST);
740 if (unlikely(ret))
741 return ret;
742 }
743 return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
744 msg.msg_controllen);
745 }
746
io_recvmsg_prep_setup(struct io_kiocb * req)747 static int io_recvmsg_prep_setup(struct io_kiocb *req)
748 {
749 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
750 struct io_async_msghdr *kmsg;
751
752 kmsg = io_msg_alloc_async(req);
753 if (unlikely(!kmsg))
754 return -ENOMEM;
755
756 if (req->opcode == IORING_OP_RECV) {
757 kmsg->msg.msg_name = NULL;
758 kmsg->msg.msg_namelen = 0;
759 kmsg->msg.msg_inq = 0;
760 kmsg->msg.msg_control = NULL;
761 kmsg->msg.msg_get_inq = 1;
762 kmsg->msg.msg_controllen = 0;
763 kmsg->msg.msg_ubuf = NULL;
764
765 if (req->flags & REQ_F_BUFFER_SELECT)
766 return 0;
767 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
768 req->flags |= REQ_F_IMPORT_BUFFER;
769 return 0;
770 }
771 return import_ubuf(ITER_DEST, sr->buf, sr->len,
772 &kmsg->msg.msg_iter);
773 }
774
775 return io_recvmsg_copy_hdr(req, kmsg);
776 }
777
778 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \
779 IORING_RECVSEND_BUNDLE | IORING_RECVSEND_FIXED_BUF)
780
io_recvmsg_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)781 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
782 {
783 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
784
785 sr->done_io = 0;
786
787 if (unlikely(sqe->addr2))
788 return -EINVAL;
789
790 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
791 sr->len = READ_ONCE(sqe->len);
792 if (unlikely(sr->len < 0))
793 return -EINVAL;
794 sr->flags = READ_ONCE(sqe->ioprio);
795 if (sr->flags & ~RECVMSG_FLAGS)
796 return -EINVAL;
797 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
798 /* registered buffer recv only for plain IORING_OP_RECV */
799 if (req->opcode != IORING_OP_RECV ||
800 (req->flags & REQ_F_BUFFER_SELECT) ||
801 (sr->flags & (IORING_RECV_MULTISHOT | IORING_RECVSEND_BUNDLE)))
802 return -EINVAL;
803 req->buf_index = READ_ONCE(sqe->buf_index);
804 }
805 sr->msg_flags = READ_ONCE(sqe->msg_flags);
806 if (sr->msg_flags & MSG_DONTWAIT)
807 req->flags |= REQ_F_NOWAIT;
808 if (sr->msg_flags & MSG_ERRQUEUE)
809 req->flags |= REQ_F_CLEAR_POLLIN;
810 if (req->flags & REQ_F_BUFFER_SELECT)
811 sr->buf_group = req->buf_index;
812 sr->mshot_total_len = sr->mshot_len = 0;
813 if (sr->flags & IORING_RECV_MULTISHOT) {
814 if (!(req->flags & REQ_F_BUFFER_SELECT))
815 return -EINVAL;
816 if (sr->msg_flags & MSG_WAITALL)
817 return -EINVAL;
818 if (req->opcode == IORING_OP_RECV) {
819 sr->mshot_len = sr->len;
820 sr->mshot_total_len = READ_ONCE(sqe->optlen);
821 if (sr->mshot_total_len)
822 sr->flags |= IORING_RECV_MSHOT_LIM;
823 } else if (sqe->optlen) {
824 return -EINVAL;
825 }
826 req->flags |= REQ_F_APOLL_MULTISHOT;
827 } else if (sqe->optlen) {
828 return -EINVAL;
829 }
830
831 if (sr->flags & IORING_RECVSEND_BUNDLE) {
832 if (req->opcode == IORING_OP_RECVMSG)
833 return -EINVAL;
834 }
835
836 if (io_is_compat(req->ctx))
837 sr->msg_flags |= MSG_CMSG_COMPAT;
838
839 sr->nr_multishot_loops = 0;
840 return io_recvmsg_prep_setup(req);
841 }
842
843 /* bits to clear in old and inherit in new cflags on bundle retry */
844 #define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE|\
845 IORING_CQE_F_BUF_MORE)
846
847 /*
848 * Finishes io_recv and io_recvmsg.
849 *
850 * Returns true if it is actually finished, or false if it should run
851 * again (for multishot).
852 */
io_recv_finish(struct io_kiocb * req,struct io_async_msghdr * kmsg,struct io_br_sel * sel,bool mshot_finished,unsigned issue_flags)853 static inline bool io_recv_finish(struct io_kiocb *req,
854 struct io_async_msghdr *kmsg,
855 struct io_br_sel *sel, bool mshot_finished,
856 unsigned issue_flags)
857 {
858 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
859 unsigned int cflags = 0;
860
861 if (kmsg->msg.msg_inq > 0)
862 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
863
864 if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) {
865 /*
866 * If sr->len hits zero, the limit has been reached. Mark
867 * mshot as finished, and flag MSHOT_DONE as well to prevent
868 * a potential bundle from being retried.
869 */
870 sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len);
871 if (!sr->mshot_total_len) {
872 sr->flags |= IORING_RECV_MSHOT_DONE;
873 mshot_finished = true;
874 }
875 }
876
877 if (sr->flags & IORING_RECVSEND_BUNDLE) {
878 size_t this_ret = sel->val - sr->done_io;
879
880 cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret));
881 if (sr->flags & IORING_RECV_RETRY)
882 cflags = req->cqe.flags | (cflags & CQE_F_MASK);
883 if (sr->mshot_len && sel->val >= sr->mshot_len)
884 sr->flags |= IORING_RECV_MSHOT_CAP;
885 /* bundle with no more immediate buffers, we're done */
886 if (req->flags & REQ_F_BL_EMPTY)
887 goto finish;
888 /*
889 * If more is available AND it was a full transfer, retry and
890 * append to this one
891 */
892 if (!(sr->flags & IORING_RECV_NO_RETRY) &&
893 kmsg->msg.msg_inq > 1 && this_ret > 0 &&
894 !iov_iter_count(&kmsg->msg.msg_iter)) {
895 req->cqe.flags = cflags & ~CQE_F_MASK;
896 sr->len = kmsg->msg.msg_inq;
897 sr->done_io += this_ret;
898 sr->flags |= IORING_RECV_RETRY;
899 return false;
900 }
901 } else {
902 cflags |= io_put_kbuf(req, sel->val, sel->buf_list);
903 }
904
905 /*
906 * Fill CQE for this receive and see if we should keep trying to
907 * receive from this socket.
908 */
909 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&
910 io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
911 sel->val = IOU_RETRY;
912 io_mshot_prep_retry(req, kmsg);
913 /* Known not-empty or unknown state, retry */
914 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {
915 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY &&
916 !(sr->flags & IORING_RECV_MSHOT_CAP)) {
917 return false;
918 }
919 /* mshot retries exceeded, force a requeue */
920 sr->nr_multishot_loops = 0;
921 sr->flags &= ~IORING_RECV_MSHOT_CAP;
922 if (issue_flags & IO_URING_F_MULTISHOT)
923 sel->val = IOU_REQUEUE;
924 }
925 return true;
926 }
927
928 /* Finish the request / stop multishot. */
929 finish:
930 io_req_set_res(req, sel->val, cflags);
931 sel->val = IOU_COMPLETE;
932 io_req_msg_cleanup(req, issue_flags);
933 return true;
934 }
935
io_recvmsg_prep_multishot(struct io_async_msghdr * kmsg,struct io_sr_msg * sr,void __user ** buf,size_t * len)936 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,
937 struct io_sr_msg *sr, void __user **buf,
938 size_t *len)
939 {
940 unsigned long ubuf = (unsigned long) *buf;
941 unsigned long hdr;
942
943 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
944 kmsg->controllen;
945 if (*len < hdr)
946 return -EFAULT;
947
948 if (kmsg->controllen) {
949 unsigned long control = ubuf + hdr - kmsg->controllen;
950
951 kmsg->msg.msg_control_user = (void __user *) control;
952 kmsg->msg.msg_controllen = kmsg->controllen;
953 }
954
955 sr->buf = *buf; /* stash for later copy */
956 *buf = (void __user *) (ubuf + hdr);
957 kmsg->payloadlen = *len = *len - hdr;
958 return 0;
959 }
960
961 struct io_recvmsg_multishot_hdr {
962 struct io_uring_recvmsg_out msg;
963 struct sockaddr_storage addr;
964 };
965
io_recvmsg_multishot(struct socket * sock,struct io_sr_msg * io,struct io_async_msghdr * kmsg,unsigned int flags,bool * finished)966 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io,
967 struct io_async_msghdr *kmsg,
968 unsigned int flags, bool *finished)
969 {
970 int err;
971 int copy_len;
972 struct io_recvmsg_multishot_hdr hdr;
973
974 if (kmsg->namelen)
975 kmsg->msg.msg_name = &hdr.addr;
976 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);
977 kmsg->msg.msg_namelen = 0;
978
979 if (sock->file->f_flags & O_NONBLOCK)
980 flags |= MSG_DONTWAIT;
981
982 err = sock_recvmsg(sock, &kmsg->msg, flags);
983 *finished = err <= 0;
984 if (err < 0)
985 return err;
986
987 hdr.msg = (struct io_uring_recvmsg_out) {
988 .controllen = kmsg->controllen - kmsg->msg.msg_controllen,
989 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT
990 };
991
992 hdr.msg.payloadlen = err;
993 if (err > kmsg->payloadlen)
994 err = kmsg->payloadlen;
995
996 copy_len = sizeof(struct io_uring_recvmsg_out);
997 if (kmsg->msg.msg_namelen > kmsg->namelen)
998 copy_len += kmsg->namelen;
999 else
1000 copy_len += kmsg->msg.msg_namelen;
1001
1002 /*
1003 * "fromlen shall refer to the value before truncation.."
1004 * 1003.1g
1005 */
1006 hdr.msg.namelen = kmsg->msg.msg_namelen;
1007
1008 /* ensure that there is no gap between hdr and sockaddr_storage */
1009 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) !=
1010 sizeof(struct io_uring_recvmsg_out));
1011 if (copy_to_user(io->buf, &hdr, copy_len)) {
1012 *finished = true;
1013 return -EFAULT;
1014 }
1015
1016 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
1017 kmsg->controllen + err;
1018 }
1019
io_recvmsg(struct io_kiocb * req,unsigned int issue_flags)1020 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
1021 {
1022 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1023 struct io_async_msghdr *kmsg = req->async_data;
1024 struct io_br_sel sel = { };
1025 struct socket *sock;
1026 unsigned flags;
1027 int ret, min_ret = 0;
1028 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1029 bool mshot_finished = true;
1030
1031 sock = sock_from_file(req->file);
1032 if (unlikely(!sock))
1033 return -ENOTSOCK;
1034
1035 if (!(req->flags & REQ_F_POLLED) &&
1036 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1037 return -EAGAIN;
1038
1039 flags = sr->msg_flags;
1040 if (force_nonblock)
1041 flags |= MSG_DONTWAIT;
1042
1043 retry_multishot:
1044 sel.buf_list = NULL;
1045 if (io_do_buffer_select(req)) {
1046 size_t len = sr->len;
1047
1048 sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1049 if (!sel.addr)
1050 return -ENOBUFS;
1051
1052 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1053 ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
1054 if (ret) {
1055 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1056 return ret;
1057 }
1058 }
1059
1060 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
1061 }
1062
1063 kmsg->msg.msg_get_inq = 1;
1064 kmsg->msg.msg_inq = -1;
1065 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1066 ret = io_recvmsg_multishot(sock, sr, kmsg, flags,
1067 &mshot_finished);
1068 } else {
1069 /* disable partial retry for recvmsg with cmsg attached */
1070 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen)
1071 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1072
1073 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,
1074 kmsg->uaddr, flags);
1075 }
1076
1077 if (ret < min_ret) {
1078 if (ret == -EAGAIN && force_nonblock) {
1079 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1080 return IOU_RETRY;
1081 }
1082 if (ret > 0 && io_net_retry(sock, flags)) {
1083 sr->done_io += ret;
1084 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
1085 }
1086 if (ret == -ERESTARTSYS)
1087 ret = -EINTR;
1088 req_set_fail(req);
1089 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1090 req_set_fail(req);
1091 }
1092
1093 if (ret > 0)
1094 ret += sr->done_io;
1095 else if (sr->done_io)
1096 ret = sr->done_io;
1097 else
1098 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1099
1100 sel.val = ret;
1101 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags))
1102 goto retry_multishot;
1103
1104 return sel.val;
1105 }
1106
io_recv_buf_select(struct io_kiocb * req,struct io_async_msghdr * kmsg,struct io_br_sel * sel,unsigned int issue_flags)1107 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
1108 struct io_br_sel *sel, unsigned int issue_flags)
1109 {
1110 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1111 int ret;
1112
1113 /*
1114 * If the ring isn't locked, then don't use the peek interface
1115 * to grab multiple buffers as we will lock/unlock between
1116 * this selection and posting the buffers.
1117 */
1118 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
1119 sr->flags & IORING_RECVSEND_BUNDLE) {
1120 struct buf_sel_arg arg = {
1121 .iovs = &kmsg->fast_iov,
1122 .nr_iovs = 1,
1123 .mode = KBUF_MODE_EXPAND,
1124 .buf_group = sr->buf_group,
1125 };
1126
1127 if (kmsg->vec.iovec) {
1128 arg.nr_iovs = kmsg->vec.nr;
1129 arg.iovs = kmsg->vec.iovec;
1130 arg.mode |= KBUF_MODE_FREE;
1131 }
1132
1133 if (sel->val)
1134 arg.max_len = sel->val;
1135 else if (kmsg->msg.msg_inq > 1)
1136 arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq);
1137
1138 /* if mshot limited, ensure we don't go over */
1139 if (sr->flags & IORING_RECV_MSHOT_LIM)
1140 arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len);
1141 ret = io_buffers_peek(req, &arg, sel);
1142 if (unlikely(ret < 0))
1143 return ret;
1144
1145 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
1146 kmsg->vec.nr = ret;
1147 kmsg->vec.iovec = arg.iovs;
1148 req->flags |= REQ_F_NEED_CLEANUP;
1149 }
1150 if (arg.partial_map)
1151 sr->flags |= IORING_RECV_PARTIAL_MAP;
1152
1153 /* special case 1 vec, can be a fast path */
1154 if (ret == 1) {
1155 sr->buf = arg.iovs[0].iov_base;
1156 sr->len = arg.iovs[0].iov_len;
1157 goto map_ubuf;
1158 }
1159 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
1160 arg.out_len);
1161 } else {
1162 size_t len = sel->val;
1163
1164 *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1165 if (!sel->addr)
1166 return -ENOBUFS;
1167 sr->buf = sel->addr;
1168 sr->len = len;
1169 map_ubuf:
1170 ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
1171 &kmsg->msg.msg_iter);
1172 if (unlikely(ret))
1173 return ret;
1174 }
1175
1176 return 0;
1177 }
1178
io_recv(struct io_kiocb * req,unsigned int issue_flags)1179 int io_recv(struct io_kiocb *req, unsigned int issue_flags)
1180 {
1181 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1182 struct io_async_msghdr *kmsg = req->async_data;
1183 struct io_br_sel sel;
1184 struct socket *sock;
1185 unsigned flags;
1186 int ret, min_ret = 0;
1187 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1188 bool mshot_finished;
1189
1190 sock = sock_from_file(req->file);
1191 if (unlikely(!sock))
1192 return -ENOTSOCK;
1193
1194 if (!(req->flags & REQ_F_POLLED) &&
1195 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1196 return -EAGAIN;
1197
1198 flags = sr->msg_flags;
1199 if (force_nonblock)
1200 flags |= MSG_DONTWAIT;
1201
1202 if (req->flags & REQ_F_IMPORT_BUFFER) {
1203 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter,
1204 (u64)(uintptr_t)sr->buf, sr->len,
1205 ITER_DEST, issue_flags);
1206 if (unlikely(ret)) {
1207 kmsg->msg.msg_inq = -1;
1208 sel.buf_list = NULL;
1209 goto out_free;
1210 }
1211 req->flags &= ~REQ_F_IMPORT_BUFFER;
1212 }
1213
1214 retry_multishot:
1215 sel.buf_list = NULL;
1216 if (io_do_buffer_select(req)) {
1217 sel.val = sr->len;
1218 ret = io_recv_buf_select(req, kmsg, &sel, issue_flags);
1219 if (unlikely(ret < 0)) {
1220 kmsg->msg.msg_inq = -1;
1221 goto out_free;
1222 }
1223 sr->buf = NULL;
1224 }
1225
1226 kmsg->msg.msg_flags = 0;
1227 kmsg->msg.msg_inq = -1;
1228
1229 if (flags & MSG_WAITALL)
1230 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1231
1232 ret = sock_recvmsg(sock, &kmsg->msg, flags);
1233 if (ret < min_ret) {
1234 if (ret == -EAGAIN && force_nonblock) {
1235 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1236 return IOU_RETRY;
1237 }
1238 if (ret > 0 && io_net_retry(sock, flags)) {
1239 sr->len -= ret;
1240 sr->buf += ret;
1241 sr->done_io += ret;
1242 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
1243 }
1244 if (ret == -ERESTARTSYS)
1245 ret = -EINTR;
1246 req_set_fail(req);
1247 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1248 out_free:
1249 req_set_fail(req);
1250 }
1251
1252 mshot_finished = ret <= 0;
1253 if (ret > 0)
1254 ret += sr->done_io;
1255 else if (sr->done_io)
1256 ret = sr->done_io;
1257 else
1258 io_kbuf_recycle(req, sel.buf_list, issue_flags);
1259
1260 sel.val = ret;
1261 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags))
1262 goto retry_multishot;
1263
1264 return sel.val;
1265 }
1266
io_recvzc_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1267 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1268 {
1269 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);
1270 unsigned ifq_idx;
1271
1272 if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3))
1273 return -EINVAL;
1274
1275 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx);
1276 zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx);
1277 if (!zc->ifq)
1278 return -EINVAL;
1279
1280 zc->len = READ_ONCE(sqe->len);
1281 zc->flags = READ_ONCE(sqe->ioprio);
1282 if (READ_ONCE(sqe->msg_flags))
1283 return -EINVAL;
1284 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT))
1285 return -EINVAL;
1286 /* multishot required */
1287 if (!(zc->flags & IORING_RECV_MULTISHOT))
1288 return -EINVAL;
1289 /* All data completions are posted as aux CQEs. */
1290 req->flags |= REQ_F_APOLL_MULTISHOT;
1291
1292 return 0;
1293 }
1294
io_recvzc(struct io_kiocb * req,unsigned int issue_flags)1295 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags)
1296 {
1297 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);
1298 struct socket *sock;
1299 unsigned int len;
1300 int ret;
1301
1302 sock = sock_from_file(req->file);
1303 if (unlikely(!sock))
1304 return -ENOTSOCK;
1305
1306 if (!(req->flags & REQ_F_POLLED) &&
1307 (zc->flags & IORING_RECVSEND_POLL_FIRST))
1308 return -EAGAIN;
1309
1310 len = zc->len;
1311 ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len);
1312 if (len && zc->len == 0) {
1313 io_req_set_res(req, 0, 0);
1314
1315 return IOU_COMPLETE;
1316 }
1317 if (unlikely(ret <= 0) && ret != -EAGAIN) {
1318 if (ret == -ERESTARTSYS)
1319 ret = -EINTR;
1320 if (ret == IOU_REQUEUE)
1321 return IOU_REQUEUE;
1322
1323 req_set_fail(req);
1324 io_req_set_res(req, ret, 0);
1325 return IOU_COMPLETE;
1326 }
1327 return IOU_RETRY;
1328 }
1329
io_send_zc_cleanup(struct io_kiocb * req)1330 void io_send_zc_cleanup(struct io_kiocb *req)
1331 {
1332 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1333 struct io_async_msghdr *io = req->async_data;
1334
1335 if (req_has_async_data(req))
1336 io_netmsg_iovec_free(io);
1337 if (zc->notif) {
1338 io_notif_flush(zc->notif);
1339 zc->notif = NULL;
1340 }
1341 }
1342
1343 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF)
1344 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \
1345 IORING_SEND_VECTORIZED)
1346
io_send_zc_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1347 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1348 {
1349 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1350 struct io_ring_ctx *ctx = req->ctx;
1351 struct io_async_msghdr *iomsg;
1352 struct io_kiocb *notif;
1353 u64 user_data;
1354 int ret;
1355
1356 zc->done_io = 0;
1357
1358 if (unlikely(READ_ONCE(sqe->__pad2[0])))
1359 return -EINVAL;
1360 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
1361 if (req->flags & REQ_F_CQE_SKIP)
1362 return -EINVAL;
1363
1364 notif = zc->notif = io_alloc_notif(ctx);
1365 if (!notif)
1366 return -ENOMEM;
1367 user_data = READ_ONCE(sqe->addr3);
1368 if (!user_data)
1369 user_data = req->cqe.user_data;
1370
1371 notif->cqe.user_data = user_data;
1372 notif->cqe.res = 0;
1373 notif->cqe.flags = IORING_CQE_F_NOTIF;
1374 req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY;
1375
1376 zc->flags = READ_ONCE(sqe->ioprio);
1377 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) {
1378 if (zc->flags & ~IO_ZC_FLAGS_VALID)
1379 return -EINVAL;
1380 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) {
1381 struct io_notif_data *nd = io_notif_to_data(notif);
1382
1383 nd->zc_report = true;
1384 nd->zc_used = false;
1385 nd->zc_copied = false;
1386 }
1387 }
1388
1389 zc->len = READ_ONCE(sqe->len);
1390 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY;
1391 req->buf_index = READ_ONCE(sqe->buf_index);
1392 if (zc->msg_flags & MSG_DONTWAIT)
1393 req->flags |= REQ_F_NOWAIT;
1394
1395 if (io_is_compat(ctx))
1396 zc->msg_flags |= MSG_CMSG_COMPAT;
1397
1398 iomsg = io_msg_alloc_async(req);
1399 if (unlikely(!iomsg))
1400 return -ENOMEM;
1401
1402 if (req->opcode == IORING_OP_SEND_ZC) {
1403 ret = io_send_setup(req, sqe);
1404 } else {
1405 if (unlikely(sqe->addr2 || sqe->file_index))
1406 return -EINVAL;
1407 ret = io_sendmsg_setup(req, sqe);
1408 }
1409 if (unlikely(ret))
1410 return ret;
1411
1412 if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) {
1413 iomsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1414 return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count);
1415 }
1416 iomsg->msg.sg_from_iter = io_sg_from_iter;
1417 return 0;
1418 }
1419
io_sg_from_iter_iovec(struct sk_buff * skb,struct iov_iter * from,size_t length)1420 static int io_sg_from_iter_iovec(struct sk_buff *skb,
1421 struct iov_iter *from, size_t length)
1422 {
1423 skb_zcopy_downgrade_managed(skb);
1424 return zerocopy_fill_skb_from_iter(skb, from, length);
1425 }
1426
io_sg_from_iter(struct sk_buff * skb,struct iov_iter * from,size_t length)1427 static int io_sg_from_iter(struct sk_buff *skb,
1428 struct iov_iter *from, size_t length)
1429 {
1430 struct skb_shared_info *shinfo = skb_shinfo(skb);
1431 int frag = shinfo->nr_frags;
1432 int ret = 0;
1433 struct bvec_iter bi;
1434 ssize_t copied = 0;
1435 unsigned long truesize = 0;
1436
1437 if (!frag)
1438 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
1439 else if (unlikely(!skb_zcopy_managed(skb)))
1440 return zerocopy_fill_skb_from_iter(skb, from, length);
1441
1442 bi.bi_size = min(from->count, length);
1443 bi.bi_offset = from->iov_offset;
1444 bi.bi_idx = 0;
1445
1446 while (bi.bi_size && frag < MAX_SKB_FRAGS) {
1447 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi);
1448
1449 copied += v.bv_len;
1450 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset);
1451 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page,
1452 v.bv_offset, v.bv_len);
1453 bvec_iter_advance_single(from->bvec, &bi, v.bv_len);
1454 }
1455 if (bi.bi_size)
1456 ret = -EMSGSIZE;
1457
1458 shinfo->nr_frags = frag;
1459 from->bvec += bi.bi_idx;
1460 from->nr_segs -= bi.bi_idx;
1461 from->count -= copied;
1462 from->iov_offset = bi.bi_offset;
1463
1464 skb->data_len += copied;
1465 skb->len += copied;
1466 skb->truesize += truesize;
1467 return ret;
1468 }
1469
io_send_zc_import(struct io_kiocb * req,struct io_async_msghdr * kmsg,unsigned int issue_flags)1470 static int io_send_zc_import(struct io_kiocb *req,
1471 struct io_async_msghdr *kmsg,
1472 unsigned int issue_flags)
1473 {
1474 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1475 struct io_kiocb *notif = sr->notif;
1476 int ret;
1477
1478 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF));
1479
1480 notif->buf_index = req->buf_index;
1481
1482 if (!(sr->flags & IORING_SEND_VECTORIZED)) {
1483 ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter,
1484 (u64)(uintptr_t)sr->buf, sr->len,
1485 ITER_SOURCE, issue_flags);
1486 } else {
1487 unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;
1488
1489 ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter,
1490 notif, &kmsg->vec, uvec_segs,
1491 issue_flags);
1492 }
1493
1494 if (unlikely(ret))
1495 return ret;
1496 req->flags &= ~REQ_F_IMPORT_BUFFER;
1497 return 0;
1498 }
1499
io_sendmsg_zc(struct io_kiocb * req,unsigned int issue_flags)1500 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
1501 {
1502 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1503 struct io_async_msghdr *kmsg = req->async_data;
1504 struct socket *sock;
1505 unsigned msg_flags;
1506 int ret, min_ret = 0;
1507
1508 sock = sock_from_file(req->file);
1509 if (unlikely(!sock))
1510 return -ENOTSOCK;
1511 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1512 return -EOPNOTSUPP;
1513 if (!(req->flags & REQ_F_POLLED) &&
1514 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1515 return -EAGAIN;
1516
1517 if (req->flags & REQ_F_IMPORT_BUFFER) {
1518 ret = io_send_zc_import(req, kmsg, issue_flags);
1519 if (unlikely(ret))
1520 return ret;
1521 }
1522
1523 msg_flags = sr->msg_flags;
1524 if (issue_flags & IO_URING_F_NONBLOCK)
1525 msg_flags |= MSG_DONTWAIT;
1526 if (msg_flags & MSG_WAITALL)
1527 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1528
1529 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1530
1531 if (req->opcode == IORING_OP_SEND_ZC) {
1532 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
1533 kmsg->msg.msg_flags = msg_flags;
1534 ret = sock_sendmsg(sock, &kmsg->msg);
1535 } else {
1536 kmsg->msg.msg_control_user = sr->msg_control;
1537 ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags);
1538 }
1539
1540 if (unlikely(ret < min_ret)) {
1541 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1542 return -EAGAIN;
1543
1544 if (ret > 0 && io_net_retry(sock, sr->msg_flags)) {
1545 sr->done_io += ret;
1546 return -EAGAIN;
1547 }
1548 if (ret == -ERESTARTSYS)
1549 ret = -EINTR;
1550 req_set_fail(req);
1551 }
1552
1553 if (ret >= 0)
1554 ret += sr->done_io;
1555 else if (sr->done_io)
1556 ret = sr->done_io;
1557
1558 /*
1559 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1560 * flushing notif to io_send_zc_cleanup()
1561 */
1562 if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1563 io_notif_flush(sr->notif);
1564 sr->notif = NULL;
1565 io_req_msg_cleanup(req, 0);
1566 }
1567 io_req_set_res(req, ret, IORING_CQE_F_MORE);
1568 return IOU_COMPLETE;
1569 }
1570
io_sendrecv_fail(struct io_kiocb * req)1571 void io_sendrecv_fail(struct io_kiocb *req)
1572 {
1573 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1574
1575 if (sr->done_io)
1576 req->cqe.res = sr->done_io;
1577
1578 if ((req->flags & REQ_F_NEED_CLEANUP) &&
1579 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC))
1580 req->cqe.flags |= IORING_CQE_F_MORE;
1581 }
1582
1583 #define ACCEPT_FLAGS (IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \
1584 IORING_ACCEPT_POLL_FIRST)
1585
io_accept_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1586 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1587 {
1588 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1589
1590 if (sqe->len || sqe->buf_index)
1591 return -EINVAL;
1592
1593 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1594 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
1595 accept->flags = READ_ONCE(sqe->accept_flags);
1596 accept->nofile = rlimit(RLIMIT_NOFILE);
1597 accept->iou_flags = READ_ONCE(sqe->ioprio);
1598 if (accept->iou_flags & ~ACCEPT_FLAGS)
1599 return -EINVAL;
1600
1601 accept->file_slot = READ_ONCE(sqe->file_index);
1602 if (accept->file_slot) {
1603 if (accept->flags & SOCK_CLOEXEC)
1604 return -EINVAL;
1605 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT &&
1606 accept->file_slot != IORING_FILE_INDEX_ALLOC)
1607 return -EINVAL;
1608 }
1609 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1610 return -EINVAL;
1611 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
1612 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
1613 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT)
1614 req->flags |= REQ_F_APOLL_MULTISHOT;
1615 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT)
1616 req->flags |= REQ_F_NOWAIT;
1617 return 0;
1618 }
1619
io_accept(struct io_kiocb * req,unsigned int issue_flags)1620 int io_accept(struct io_kiocb *req, unsigned int issue_flags)
1621 {
1622 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1623 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1624 bool fixed = !!accept->file_slot;
1625 struct proto_accept_arg arg = {
1626 .flags = force_nonblock ? O_NONBLOCK : 0,
1627 };
1628 struct file *file;
1629 unsigned cflags;
1630 int ret, fd;
1631
1632 if (!(req->flags & REQ_F_POLLED) &&
1633 accept->iou_flags & IORING_ACCEPT_POLL_FIRST)
1634 return -EAGAIN;
1635
1636 retry:
1637 if (!fixed) {
1638 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
1639 if (unlikely(fd < 0))
1640 return fd;
1641 }
1642 arg.err = 0;
1643 arg.is_empty = -1;
1644 file = do_accept(req->file, &arg, accept->addr, accept->addr_len,
1645 accept->flags);
1646 if (IS_ERR(file)) {
1647 if (!fixed)
1648 put_unused_fd(fd);
1649 ret = PTR_ERR(file);
1650 if (ret == -EAGAIN && force_nonblock &&
1651 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT))
1652 return IOU_RETRY;
1653
1654 if (ret == -ERESTARTSYS)
1655 ret = -EINTR;
1656 } else if (!fixed) {
1657 fd_install(fd, file);
1658 ret = fd;
1659 } else {
1660 ret = io_fixed_fd_install(req, issue_flags, file,
1661 accept->file_slot);
1662 }
1663
1664 cflags = 0;
1665 if (!arg.is_empty)
1666 cflags |= IORING_CQE_F_SOCK_NONEMPTY;
1667
1668 if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) &&
1669 io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
1670 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1)
1671 goto retry;
1672 return IOU_RETRY;
1673 }
1674
1675 io_req_set_res(req, ret, cflags);
1676 if (ret < 0)
1677 req_set_fail(req);
1678 return IOU_COMPLETE;
1679 }
1680
io_socket_bpf_populate(struct io_uring_bpf_ctx * bctx,struct io_kiocb * req)1681 void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
1682 {
1683 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1684
1685 bctx->socket.family = sock->domain;
1686 bctx->socket.type = sock->type;
1687 bctx->socket.protocol = sock->protocol;
1688 }
1689
io_connect_bpf_populate(struct io_uring_bpf_ctx * bctx,struct io_kiocb * req)1690 void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
1691 {
1692 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
1693 struct sockaddr_storage *ss = req->async_data;
1694
1695 /*
1696 * move_addr_to_kernel() skips the copy for addr_len == 0, so
1697 * iomsg->addr may hold stale data from a prior CONNECT. Bail
1698 * unless addr_len covers the family discriminator.
1699 */
1700 if (conn->addr_len < (int)sizeof(sa_family_t))
1701 return;
1702
1703 bctx->connect.family = ss->ss_family;
1704 switch (ss->ss_family) {
1705 case AF_INET: {
1706 struct sockaddr_in *sin = (struct sockaddr_in *)ss;
1707
1708 if (conn->addr_len < (int)sizeof(*sin))
1709 break;
1710 bctx->connect.port = sin->sin_port;
1711 bctx->connect.v4_addr = sin->sin_addr.s_addr;
1712 break;
1713 }
1714 case AF_INET6: {
1715 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;
1716
1717 if (conn->addr_len < (int)sizeof(*sin6))
1718 break;
1719 bctx->connect.port = sin6->sin6_port;
1720 memcpy(bctx->connect.v6_addr, &sin6->sin6_addr,
1721 sizeof(bctx->connect.v6_addr));
1722 break;
1723 }
1724 default:
1725 /* family is set; per-family fields stay zero - family-only filtering */
1726 break;
1727 }
1728 }
1729
io_socket_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1730 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1731 {
1732 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1733
1734 if (sqe->addr || sqe->rw_flags || sqe->buf_index)
1735 return -EINVAL;
1736
1737 sock->domain = READ_ONCE(sqe->fd);
1738 sock->type = READ_ONCE(sqe->off);
1739 sock->protocol = READ_ONCE(sqe->len);
1740 sock->file_slot = READ_ONCE(sqe->file_index);
1741 sock->nofile = rlimit(RLIMIT_NOFILE);
1742
1743 sock->flags = sock->type & ~SOCK_TYPE_MASK;
1744 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
1745 return -EINVAL;
1746 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1747 return -EINVAL;
1748 return 0;
1749 }
1750
io_socket(struct io_kiocb * req,unsigned int issue_flags)1751 int io_socket(struct io_kiocb *req, unsigned int issue_flags)
1752 {
1753 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1754 bool fixed = !!sock->file_slot;
1755 struct file *file;
1756 int ret, fd;
1757
1758 if (!fixed) {
1759 fd = __get_unused_fd_flags(sock->flags, sock->nofile);
1760 if (unlikely(fd < 0))
1761 return fd;
1762 }
1763 file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
1764 if (IS_ERR(file)) {
1765 if (!fixed)
1766 put_unused_fd(fd);
1767 ret = PTR_ERR(file);
1768 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1769 return -EAGAIN;
1770 if (ret == -ERESTARTSYS)
1771 ret = -EINTR;
1772 req_set_fail(req);
1773 } else if (!fixed) {
1774 fd_install(fd, file);
1775 ret = fd;
1776 } else {
1777 ret = io_fixed_fd_install(req, issue_flags, file,
1778 sock->file_slot);
1779 }
1780 io_req_set_res(req, ret, 0);
1781 return IOU_COMPLETE;
1782 }
1783
io_connect_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1784 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1785 {
1786 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
1787 struct sockaddr_storage *addr;
1788
1789 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1790 return -EINVAL;
1791
1792 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1793 conn->addr_len = READ_ONCE(sqe->addr2);
1794 conn->in_progress = conn->seen_econnaborted = false;
1795
1796 addr = io_uring_alloc_async_data(NULL, req);
1797 if (unlikely(!addr))
1798 return -ENOMEM;
1799
1800 return move_addr_to_kernel(conn->addr, conn->addr_len, addr);
1801 }
1802
io_connect(struct io_kiocb * req,unsigned int issue_flags)1803 int io_connect(struct io_kiocb *req, unsigned int issue_flags)
1804 {
1805 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect);
1806 struct sockaddr_storage *addr = req->async_data;
1807 unsigned file_flags;
1808 int ret;
1809 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1810
1811 if (connect->in_progress) {
1812 struct poll_table_struct pt = { ._key = EPOLLERR };
1813
1814 if (vfs_poll(req->file, &pt) & EPOLLERR)
1815 goto get_sock_err;
1816 }
1817
1818 file_flags = force_nonblock ? O_NONBLOCK : 0;
1819
1820 ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags);
1821 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED)
1822 && force_nonblock) {
1823 if (ret == -EINPROGRESS) {
1824 connect->in_progress = true;
1825 } else if (ret == -ECONNABORTED) {
1826 if (connect->seen_econnaborted)
1827 goto out;
1828 connect->seen_econnaborted = true;
1829 }
1830 return -EAGAIN;
1831 }
1832 if (connect->in_progress) {
1833 /*
1834 * At least bluetooth will return -EBADFD on a re-connect
1835 * attempt, and it's (supposedly) also valid to get -EISCONN
1836 * which means the previous result is good. For both of these,
1837 * grab the sock_error() and use that for the completion.
1838 */
1839 if (ret == -EBADFD || ret == -EISCONN) {
1840 get_sock_err:
1841 ret = sock_error(sock_from_file(req->file)->sk);
1842 }
1843 }
1844 if (ret == -ERESTARTSYS)
1845 ret = -EINTR;
1846 out:
1847 if (ret < 0)
1848 req_set_fail(req);
1849 io_req_set_res(req, ret, 0);
1850 return IOU_COMPLETE;
1851 }
1852
1853 /*
1854 * Check if bind request would potentially end up with filename_create(),
1855 * which in turn end up in mnt_want_write() which will grab the fs
1856 * percpu start write sem. This can trigger a lockdep warning.
1857 */
io_bind_file_create(const struct sockaddr_storage * addr,int addr_len)1858 static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len)
1859 {
1860 const struct sockaddr_un *sun;
1861
1862 if (addr->ss_family != AF_UNIX)
1863 return 0;
1864 if (addr_len <= offsetof(struct sockaddr_un, sun_path))
1865 return 0;
1866 sun = (const struct sockaddr_un *) addr;
1867 return sun->sun_path[0] != '\0';
1868 }
1869
io_bind_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1870 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1871 {
1872 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1873 struct sockaddr __user *uaddr;
1874 struct sockaddr_storage *addr;
1875 int ret;
1876
1877 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1878 return -EINVAL;
1879
1880 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1881 bind->addr_len = READ_ONCE(sqe->addr2);
1882
1883 addr = io_uring_alloc_async_data(NULL, req);
1884 if (unlikely(!addr))
1885 return -ENOMEM;
1886 ret = move_addr_to_kernel(uaddr, bind->addr_len, addr);
1887 if (unlikely(ret))
1888 return ret;
1889 if (io_bind_file_create(addr, bind->addr_len))
1890 req->flags |= REQ_F_FORCE_ASYNC;
1891 return 0;
1892 }
1893
1894
io_bind(struct io_kiocb * req,unsigned int issue_flags)1895 int io_bind(struct io_kiocb *req, unsigned int issue_flags)
1896 {
1897 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1898 struct sockaddr_storage *addr = req->async_data;
1899 struct socket *sock;
1900 int ret;
1901
1902 sock = sock_from_file(req->file);
1903 if (unlikely(!sock))
1904 return -ENOTSOCK;
1905
1906 ret = __sys_bind_socket(sock, addr, bind->addr_len);
1907 if (ret < 0)
1908 req_set_fail(req);
1909 io_req_set_res(req, ret, 0);
1910 return 0;
1911 }
1912
io_listen_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)1913 int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1914 {
1915 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1916
1917 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2)
1918 return -EINVAL;
1919
1920 listen->backlog = READ_ONCE(sqe->len);
1921 return 0;
1922 }
1923
io_listen(struct io_kiocb * req,unsigned int issue_flags)1924 int io_listen(struct io_kiocb *req, unsigned int issue_flags)
1925 {
1926 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1927 struct socket *sock;
1928 int ret;
1929
1930 sock = sock_from_file(req->file);
1931 if (unlikely(!sock))
1932 return -ENOTSOCK;
1933
1934 ret = __sys_listen_socket(sock, listen->backlog);
1935 if (ret < 0)
1936 req_set_fail(req);
1937 io_req_set_res(req, ret, 0);
1938 return 0;
1939 }
1940
io_netmsg_cache_free(const void * entry)1941 void io_netmsg_cache_free(const void *entry)
1942 {
1943 struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry;
1944
1945 io_vec_free(&kmsg->vec);
1946 kfree(kmsg);
1947 }
1948