xref: /linux/io_uring/net.c (revision 3e3aebb7570ccdde5cc49c9bfc8fd3a436b71b0c)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/slab.h>
6 #include <linux/net.h>
7 #include <linux/un.h>
8 #include <linux/compat.h>
9 #include <net/compat.h>
10 #include <net/ipv6.h>
11 #include <linux/io_uring.h>
12 
13 #include <uapi/linux/io_uring.h>
14 
15 #include "filetable.h"
16 #include "io_uring.h"
17 #include "kbuf.h"
18 #include "alloc_cache.h"
19 #include "net.h"
20 #include "notif.h"
21 #include "rsrc.h"
22 #include "zcrx.h"
23 
24 struct io_shutdown {
25 	struct file			*file;
26 	int				how;
27 };
28 
29 struct io_accept {
30 	struct file			*file;
31 	struct sockaddr __user		*addr;
32 	int __user			*addr_len;
33 	int				flags;
34 	int				iou_flags;
35 	u32				file_slot;
36 	unsigned long			nofile;
37 };
38 
39 struct io_socket {
40 	struct file			*file;
41 	int				domain;
42 	int				type;
43 	int				protocol;
44 	int				flags;
45 	u32				file_slot;
46 	unsigned long			nofile;
47 };
48 
49 struct io_connect {
50 	struct file			*file;
51 	struct sockaddr __user		*addr;
52 	int				addr_len;
53 	bool				in_progress;
54 	bool				seen_econnaborted;
55 };
56 
57 struct io_bind {
58 	struct file			*file;
59 	int				addr_len;
60 };
61 
62 struct io_listen {
63 	struct file			*file;
64 	int				backlog;
65 };
66 
67 struct io_sr_msg {
68 	struct file			*file;
69 	union {
70 		struct compat_msghdr __user	*umsg_compat;
71 		struct user_msghdr __user	*umsg;
72 		void __user			*buf;
73 	};
74 	int				len;
75 	unsigned			done_io;
76 	unsigned			msg_flags;
77 	unsigned			nr_multishot_loops;
78 	u16				flags;
79 	/* initialised and used only by !msg send variants */
80 	u16				buf_group;
81 	/* per-invocation mshot limit */
82 	unsigned			mshot_len;
83 	/* overall mshot byte limit */
84 	unsigned			mshot_total_len;
85 	void __user			*msg_control;
86 	/* used only for send zerocopy */
87 	struct io_kiocb 		*notif;
88 };
89 
90 /*
91  * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold
92  * anyway. Use the upper 8 bits for internal uses.
93  */
94 enum sr_retry_flags {
95 	IORING_RECV_RETRY	= (1U << 15),
96 	IORING_RECV_PARTIAL_MAP	= (1U << 14),
97 	IORING_RECV_MSHOT_CAP	= (1U << 13),
98 	IORING_RECV_MSHOT_LIM	= (1U << 12),
99 	IORING_RECV_MSHOT_DONE	= (1U << 11),
100 
101 	IORING_RECV_RETRY_CLEAR	= IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP,
102 	IORING_RECV_NO_RETRY	= IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP |
103 				  IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE,
104 };
105 
106 /*
107  * Number of times we'll try and do receives if there's more data. If we
108  * exceed this limit, then add us to the back of the queue and retry from
109  * there. This helps fairness between flooding clients.
110  */
111 #define MULTISHOT_MAX_RETRY	32
112 
113 struct io_recvzc {
114 	struct file			*file;
115 	u16				flags;
116 	u32				len;
117 	struct io_zcrx_ifq		*ifq;
118 };
119 
120 static int io_sg_from_iter_iovec(struct sk_buff *skb,
121 				 struct iov_iter *from, size_t length);
122 static int io_sg_from_iter(struct sk_buff *skb,
123 			   struct iov_iter *from, size_t length);
124 
125 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
126 {
127 	struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
128 
129 	if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
130 		     sqe->buf_index || sqe->splice_fd_in))
131 		return -EINVAL;
132 
133 	shutdown->how = READ_ONCE(sqe->len);
134 	req->flags |= REQ_F_FORCE_ASYNC;
135 	return 0;
136 }
137 
138 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
139 {
140 	struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown);
141 	struct socket *sock;
142 	int ret;
143 
144 	WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
145 
146 	sock = sock_from_file(req->file);
147 	if (unlikely(!sock))
148 		return -ENOTSOCK;
149 
150 	ret = __sys_shutdown_sock(sock, shutdown->how);
151 	io_req_set_res(req, ret, 0);
152 	return IOU_COMPLETE;
153 }
154 
155 static bool io_net_retry(struct socket *sock, int flags)
156 {
157 	if (!(flags & MSG_WAITALL))
158 		return false;
159 	return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
160 }
161 
162 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)
163 {
164 	if (kmsg->vec.iovec)
165 		io_vec_free(&kmsg->vec);
166 }
167 
168 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
169 {
170 	struct io_async_msghdr *hdr = req->async_data;
171 
172 	/* can't recycle, ensure we free the iovec if we have one */
173 	if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
174 		io_netmsg_iovec_free(hdr);
175 		return;
176 	}
177 
178 	/* Let normal cleanup path reap it if we fail adding to the cache */
179 	io_alloc_cache_vec_kasan(&hdr->vec);
180 	if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP)
181 		io_vec_free(&hdr->vec);
182 
183 	if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr))
184 		io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
185 }
186 
187 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
188 {
189 	struct io_ring_ctx *ctx = req->ctx;
190 	struct io_async_msghdr *hdr;
191 
192 	hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req);
193 	if (!hdr)
194 		return NULL;
195 
196 	/* If the async data was cached, we might have an iov cached inside. */
197 	if (hdr->vec.iovec)
198 		req->flags |= REQ_F_NEED_CLEANUP;
199 	return hdr;
200 }
201 
202 static inline void io_mshot_prep_retry(struct io_kiocb *req,
203 				       struct io_async_msghdr *kmsg)
204 {
205 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
206 
207 	req->flags &= ~REQ_F_BL_EMPTY;
208 	sr->done_io = 0;
209 	sr->flags &= ~IORING_RECV_RETRY_CLEAR;
210 	sr->len = sr->mshot_len;
211 }
212 
213 static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg,
214 			     const struct iovec __user *uiov, unsigned uvec_seg,
215 			     int ddir)
216 {
217 	struct iovec *iov;
218 	int ret, nr_segs;
219 
220 	if (iomsg->vec.iovec) {
221 		nr_segs = iomsg->vec.nr;
222 		iov = iomsg->vec.iovec;
223 	} else {
224 		nr_segs = 1;
225 		iov = &iomsg->fast_iov;
226 	}
227 
228 	ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov,
229 			     &iomsg->msg.msg_iter, io_is_compat(req->ctx));
230 	if (unlikely(ret < 0))
231 		return ret;
232 
233 	if (iov) {
234 		req->flags |= REQ_F_NEED_CLEANUP;
235 		io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs);
236 	}
237 	return 0;
238 }
239 
240 static int io_copy_msghdr_from_user(struct user_msghdr *msg,
241 				    struct user_msghdr __user *umsg)
242 {
243 	if (!user_access_begin(umsg, sizeof(*umsg)))
244 		return -EFAULT;
245 	unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end);
246 	unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end);
247 	unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end);
248 	unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end);
249 	unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end);
250 	unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end);
251 	user_access_end();
252 	return 0;
253 ua_end:
254 	user_access_end();
255 	return -EFAULT;
256 }
257 
258 static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
259 			   struct user_msghdr *msg,
260 			   struct sockaddr __user **save_addr)
261 {
262 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
263 	int ret;
264 
265 	iomsg->msg.msg_name = &iomsg->addr;
266 	iomsg->msg.msg_iter.nr_segs = 0;
267 
268 	if (io_is_compat(req->ctx)) {
269 		struct compat_msghdr cmsg;
270 
271 		if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg)))
272 			return -EFAULT;
273 
274 		ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr);
275 		if (ret)
276 			return ret;
277 
278 		memset(msg, 0, sizeof(*msg));
279 		msg->msg_namelen = cmsg.msg_namelen;
280 		msg->msg_controllen = cmsg.msg_controllen;
281 		msg->msg_iov = compat_ptr(cmsg.msg_iov);
282 		msg->msg_iovlen = cmsg.msg_iovlen;
283 	} else {
284 		ret = io_copy_msghdr_from_user(msg, sr->umsg);
285 		if (unlikely(ret))
286 			return ret;
287 
288 		msg->msg_flags = 0;
289 
290 		ret = __copy_msghdr(&iomsg->msg, msg, save_addr);
291 		if (ret)
292 			return ret;
293 	}
294 
295 	if (req->flags & REQ_F_BUFFER_SELECT) {
296 		if (msg->msg_iovlen == 0) {
297 			sr->len = 0;
298 		} else if (msg->msg_iovlen > 1) {
299 			return -EINVAL;
300 		} else {
301 			struct iovec fast_iov, *iov;
302 
303 			iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov,
304 					      io_is_compat(req->ctx));
305 			if (IS_ERR(iov))
306 				return PTR_ERR(iov);
307 			sr->len = iov->iov_len;
308 		}
309 	}
310 	return 0;
311 }
312 
313 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
314 {
315 	struct io_async_msghdr *io = req->async_data;
316 
317 	io_netmsg_iovec_free(io);
318 }
319 
320 static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
321 {
322 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
323 	struct io_async_msghdr *kmsg = req->async_data;
324 	void __user *addr;
325 	u16 addr_len;
326 	int ret;
327 
328 	sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
329 
330 	if (READ_ONCE(sqe->__pad3[0]))
331 		return -EINVAL;
332 
333 	kmsg->msg.msg_name = NULL;
334 	kmsg->msg.msg_namelen = 0;
335 	kmsg->msg.msg_control = NULL;
336 	kmsg->msg.msg_controllen = 0;
337 	kmsg->msg.msg_ubuf = NULL;
338 
339 	addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
340 	addr_len = READ_ONCE(sqe->addr_len);
341 	if (addr) {
342 		ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);
343 		if (unlikely(ret < 0))
344 			return ret;
345 		kmsg->msg.msg_name = &kmsg->addr;
346 		kmsg->msg.msg_namelen = addr_len;
347 	}
348 	if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
349 		if (!(sr->flags & IORING_SEND_VECTORIZED)) {
350 			req->flags |= REQ_F_IMPORT_BUFFER;
351 			return 0;
352 		}
353 
354 		kmsg->msg.msg_iter.nr_segs = sr->len;
355 		return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len);
356 	}
357 	if (req->flags & REQ_F_BUFFER_SELECT)
358 		return 0;
359 
360 	if (sr->flags & IORING_SEND_VECTORIZED)
361 		return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE);
362 
363 	return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);
364 }
365 
366 static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe)
367 {
368 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
369 	struct io_async_msghdr *kmsg = req->async_data;
370 	struct user_msghdr msg;
371 	int ret;
372 
373 	sr->flags |= IORING_SEND_VECTORIZED;
374 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
375 	ret = io_msg_copy_hdr(req, kmsg, &msg, NULL);
376 	if (unlikely(ret))
377 		return ret;
378 	/* save msg_control as sys_sendmsg() overwrites it */
379 	sr->msg_control = kmsg->msg.msg_control_user;
380 
381 	if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
382 		kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen;
383 		return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov,
384 					 msg.msg_iovlen);
385 	}
386 	if (req->flags & REQ_F_BUFFER_SELECT)
387 		return 0;
388 	return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE);
389 }
390 
391 #define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | \
392 			IORING_SEND_VECTORIZED | IORING_RECVSEND_FIXED_BUF)
393 
394 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
395 {
396 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
397 
398 	sr->done_io = 0;
399 	sr->len = READ_ONCE(sqe->len);
400 	if (unlikely(sr->len < 0))
401 		return -EINVAL;
402 	sr->flags = READ_ONCE(sqe->ioprio);
403 	if (sr->flags & ~SENDMSG_FLAGS)
404 		return -EINVAL;
405 	if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
406 		/* registered buffer send only supported for plain IORING_OP_SEND */
407 		if (req->opcode != IORING_OP_SEND ||
408 		    (req->flags & REQ_F_BUFFER_SELECT) ||
409 		    (sr->flags & (IORING_RECVSEND_BUNDLE|IORING_SEND_VECTORIZED)))
410 			return -EINVAL;
411 		req->buf_index = READ_ONCE(sqe->buf_index);
412 	}
413 	sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
414 	if (sr->msg_flags & MSG_DONTWAIT)
415 		req->flags |= REQ_F_NOWAIT;
416 	if (req->flags & REQ_F_BUFFER_SELECT)
417 		sr->buf_group = req->buf_index;
418 	sr->mshot_total_len = sr->mshot_len = 0;
419 	if (sr->flags & IORING_RECVSEND_BUNDLE) {
420 		if (req->opcode == IORING_OP_SENDMSG)
421 			return -EINVAL;
422 		sr->msg_flags |= MSG_WAITALL;
423 		req->flags |= REQ_F_MULTISHOT;
424 	}
425 
426 	if (io_is_compat(req->ctx))
427 		sr->msg_flags |= MSG_CMSG_COMPAT;
428 
429 	if (unlikely(!io_msg_alloc_async(req)))
430 		return -ENOMEM;
431 	if (req->opcode != IORING_OP_SENDMSG)
432 		return io_send_setup(req, sqe);
433 	if (unlikely(sqe->addr2 || sqe->file_index))
434 		return -EINVAL;
435 	return io_sendmsg_setup(req, sqe);
436 }
437 
438 static void io_req_msg_cleanup(struct io_kiocb *req,
439 			       unsigned int issue_flags)
440 {
441 	io_netmsg_recycle(req, issue_flags);
442 }
443 
444 /*
445  * For bundle completions, we need to figure out how many segments we consumed.
446  * A bundle could be using a single ITER_UBUF if that's all we mapped, or it
447  * could be using an ITER_IOVEC. If the latter, then if we consumed all of
448  * the segments, then it's a trivial questiont o answer. If we have residual
449  * data in the iter, then loop the segments to figure out how much we
450  * transferred.
451  */
452 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)
453 {
454 	struct iovec *iov;
455 	int nbufs;
456 
457 	/* no data is always zero segments, and a ubuf is always 1 segment */
458 	if (ret <= 0)
459 		return 0;
460 	if (iter_is_ubuf(&kmsg->msg.msg_iter))
461 		return 1;
462 
463 	iov = kmsg->vec.iovec;
464 	if (!iov)
465 		iov = &kmsg->fast_iov;
466 
467 	/* if all data was transferred, it's basic pointer math */
468 	if (!iov_iter_count(&kmsg->msg.msg_iter))
469 		return iter_iov(&kmsg->msg.msg_iter) - iov;
470 
471 	/* short transfer, count segments */
472 	nbufs = 0;
473 	do {
474 		int this_len = min_t(int, iov[nbufs].iov_len, ret);
475 
476 		nbufs++;
477 		ret -= this_len;
478 	} while (ret);
479 
480 	return nbufs;
481 }
482 
483 /*
484  * A receive that is retried after a partial transfer keeps writing to what it
485  * has mapped. io_net_kbuf_recyle() only commits what was filled so far, which
486  * takes a buffer out of circulation if it's a single one that is consumed as a
487  * whole. The tail of an incrementally consumed buffer, or the yet unused
488  * buffers of a bundle, would be up for grabs. They can't be committed in
489  * advance either, the CQE has no way to tell if they end up not being used.
490  */
491 static bool io_net_kbuf_can_retry(struct io_kiocb *req,
492 				  struct io_buffer_list *bl,
493 				  struct io_async_msghdr *kmsg)
494 {
495 	/* not ring provided, or committed at selection time */
496 	if (!(req->flags & REQ_F_BUFFERS_COMMIT))
497 		return true;
498 	return !(bl->flags & IOBL_INC) && iter_is_ubuf(&kmsg->msg.msg_iter);
499 }
500 
501 static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl,
502 			      struct io_async_msghdr *kmsg, int len)
503 {
504 	req->flags |= REQ_F_BL_NO_RECYCLE;
505 	if (req->flags & REQ_F_BUFFERS_COMMIT)
506 		io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len));
507 	return IOU_RETRY;
508 }
509 
510 static inline bool io_send_finish(struct io_kiocb *req,
511 				  struct io_async_msghdr *kmsg,
512 				  struct io_br_sel *sel)
513 {
514 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
515 	bool bundle_finished = sel->val <= 0;
516 	unsigned int cflags;
517 
518 	if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {
519 		cflags = io_put_kbuf(req, sel->val, sel->buf_list);
520 		goto finish;
521 	}
522 
523 	cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val));
524 
525 	/*
526 	 * Don't start new bundles if the buffer list is empty, or if the
527 	 * current operation needed to go through polling to complete.
528 	 */
529 	if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED))
530 		goto finish;
531 
532 	/*
533 	 * Fill CQE for this receive and see if we should keep trying to
534 	 * receive from this socket.
535 	 */
536 	if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
537 		io_mshot_prep_retry(req, kmsg);
538 		return false;
539 	}
540 
541 	/* Otherwise stop bundle and use the current result. */
542 finish:
543 	io_req_set_res(req, sel->val, cflags);
544 	sel->val = IOU_COMPLETE;
545 	return true;
546 }
547 
548 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
549 {
550 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
551 	struct io_async_msghdr *kmsg = req->async_data;
552 	struct socket *sock;
553 	unsigned flags;
554 	int min_ret = 0;
555 	int ret;
556 
557 	sock = sock_from_file(req->file);
558 	if (unlikely(!sock))
559 		return -ENOTSOCK;
560 
561 	if (!(req->flags & REQ_F_POLLED) &&
562 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
563 		return -EAGAIN;
564 
565 	flags = sr->msg_flags;
566 	if (issue_flags & IO_URING_F_NONBLOCK)
567 		flags |= MSG_DONTWAIT;
568 	if (flags & MSG_WAITALL)
569 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);
570 
571 	kmsg->msg.msg_control_user = sr->msg_control;
572 
573 	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
574 
575 	if (ret < min_ret) {
576 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
577 			return -EAGAIN;
578 		if (ret > 0 && io_net_retry(sock, flags)) {
579 			kmsg->msg.msg_controllen = 0;
580 			kmsg->msg.msg_control = NULL;
581 			sr->done_io += ret;
582 			return -EAGAIN;
583 		}
584 		if (ret == -ERESTARTSYS)
585 			ret = -EINTR;
586 		req_set_fail(req);
587 	}
588 	io_req_msg_cleanup(req, issue_flags);
589 	if (ret >= 0)
590 		ret += sr->done_io;
591 	else if (sr->done_io)
592 		ret = sr->done_io;
593 	io_req_set_res(req, ret, 0);
594 	return IOU_COMPLETE;
595 }
596 
597 static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags,
598 				 struct io_br_sel *sel, struct io_async_msghdr *kmsg)
599 {
600 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
601 	struct buf_sel_arg arg = {
602 		.iovs = &kmsg->fast_iov,
603 		.max_len = min_not_zero(sr->len, INT_MAX),
604 		.nr_iovs = 1,
605 		.buf_group = sr->buf_group,
606 	};
607 	int ret;
608 
609 	if (kmsg->vec.iovec) {
610 		arg.nr_iovs = kmsg->vec.nr;
611 		arg.iovs = kmsg->vec.iovec;
612 		arg.mode = KBUF_MODE_FREE;
613 	}
614 
615 	if (!(sr->flags & IORING_RECVSEND_BUNDLE))
616 		arg.nr_iovs = 1;
617 	else
618 		arg.mode |= KBUF_MODE_EXPAND;
619 
620 	ret = io_buffers_select(req, &arg, sel, issue_flags);
621 	if (unlikely(ret < 0))
622 		return ret;
623 
624 	if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
625 		kmsg->vec.nr = ret;
626 		kmsg->vec.iovec = arg.iovs;
627 		req->flags |= REQ_F_NEED_CLEANUP;
628 	}
629 	sr->len = arg.out_len;
630 
631 	if (ret == 1) {
632 		sr->buf = arg.iovs[0].iov_base;
633 		ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,
634 					&kmsg->msg.msg_iter);
635 		if (unlikely(ret))
636 			return ret;
637 	} else {
638 		iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,
639 				arg.iovs, ret, arg.out_len);
640 	}
641 
642 	return 0;
643 }
644 
645 int io_send(struct io_kiocb *req, unsigned int issue_flags)
646 {
647 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
648 	struct io_async_msghdr *kmsg = req->async_data;
649 	struct io_br_sel sel = { };
650 	struct socket *sock;
651 	unsigned flags;
652 	int min_ret = 0;
653 	int ret;
654 
655 	sock = sock_from_file(req->file);
656 	if (unlikely(!sock))
657 		return -ENOTSOCK;
658 
659 	if (!(req->flags & REQ_F_POLLED) &&
660 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
661 		return -EAGAIN;
662 
663 	if (req->flags & REQ_F_IMPORT_BUFFER) {
664 		ret = io_import_reg_buf(req, &kmsg->msg.msg_iter,
665 					(u64)(uintptr_t)sr->buf, sr->len,
666 					ITER_SOURCE, issue_flags);
667 		if (unlikely(ret))
668 			return ret;
669 		req->flags &= ~REQ_F_IMPORT_BUFFER;
670 	}
671 
672 	flags = sr->msg_flags;
673 	if (issue_flags & IO_URING_F_NONBLOCK)
674 		flags |= MSG_DONTWAIT;
675 
676 retry_bundle:
677 	sel.buf_list = NULL;
678 	if (io_do_buffer_select(req)) {
679 		ret = io_send_select_buffer(req, issue_flags, &sel, kmsg);
680 		if (ret)
681 			return ret;
682 	}
683 
684 	/*
685 	 * If MSG_WAITALL is set, or this is a bundle send, then we need
686 	 * the full amount. If just bundle is set, if we do a short send
687 	 * then we complete the bundle sequence rather than continue on.
688 	 */
689 	if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE)
690 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);
691 
692 	flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
693 	kmsg->msg.msg_flags = flags;
694 	ret = sock_sendmsg(sock, &kmsg->msg);
695 	if (ret < min_ret) {
696 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
697 			return -EAGAIN;
698 
699 		if (ret > 0 && io_net_retry(sock, flags)) {
700 			sr->len -= ret;
701 			sr->buf += ret;
702 			sr->done_io += ret;
703 			return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
704 		}
705 		if (ret == -ERESTARTSYS)
706 			ret = -EINTR;
707 		req_set_fail(req);
708 	}
709 	if (ret >= 0)
710 		ret += sr->done_io;
711 	else if (sr->done_io)
712 		ret = sr->done_io;
713 
714 	sel.val = ret;
715 	if (!io_send_finish(req, kmsg, &sel))
716 		goto retry_bundle;
717 
718 	io_req_msg_cleanup(req, issue_flags);
719 	return sel.val;
720 }
721 
722 static int io_recvmsg_mshot_prep(struct io_kiocb *req,
723 				 struct io_async_msghdr *iomsg,
724 				 int namelen, size_t controllen)
725 {
726 	if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) ==
727 			  (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) {
728 		int hdr;
729 
730 		if (unlikely(namelen < 0))
731 			return -EOVERFLOW;
732 		if (check_add_overflow(sizeof(struct io_uring_recvmsg_out),
733 					namelen, &hdr))
734 			return -EOVERFLOW;
735 		if (check_add_overflow(hdr, controllen, &hdr))
736 			return -EOVERFLOW;
737 
738 		iomsg->namelen = namelen;
739 		iomsg->controllen = controllen;
740 		return 0;
741 	}
742 
743 	return 0;
744 }
745 
746 static int io_recvmsg_copy_hdr(struct io_kiocb *req,
747 			       struct io_async_msghdr *iomsg)
748 {
749 	struct user_msghdr msg;
750 	int ret;
751 
752 	ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr);
753 	if (unlikely(ret))
754 		return ret;
755 
756 	if (!(req->flags & REQ_F_BUFFER_SELECT)) {
757 		ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen,
758 					ITER_DEST);
759 		if (unlikely(ret))
760 			return ret;
761 	}
762 	return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen,
763 					msg.msg_controllen);
764 }
765 
766 static int io_recvmsg_prep_setup(struct io_kiocb *req)
767 {
768 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
769 	struct io_async_msghdr *kmsg;
770 
771 	kmsg = io_msg_alloc_async(req);
772 	if (unlikely(!kmsg))
773 		return -ENOMEM;
774 
775 	if (req->opcode == IORING_OP_RECV) {
776 		kmsg->msg.msg_name = NULL;
777 		kmsg->msg.msg_namelen = 0;
778 		kmsg->msg.msg_inq = 0;
779 		kmsg->msg.msg_control = NULL;
780 		kmsg->msg.msg_get_inq = 1;
781 		kmsg->msg.msg_controllen = 0;
782 		kmsg->msg.msg_ubuf = NULL;
783 
784 		if (req->flags & REQ_F_BUFFER_SELECT)
785 			return 0;
786 		if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
787 			req->flags |= REQ_F_IMPORT_BUFFER;
788 			return 0;
789 		}
790 		return import_ubuf(ITER_DEST, sr->buf, sr->len,
791 				   &kmsg->msg.msg_iter);
792 	}
793 
794 	return io_recvmsg_copy_hdr(req, kmsg);
795 }
796 
797 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \
798 			IORING_RECVSEND_BUNDLE | IORING_RECVSEND_FIXED_BUF)
799 
800 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
801 {
802 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
803 
804 	sr->done_io = 0;
805 
806 	if (unlikely(sqe->addr2))
807 		return -EINVAL;
808 
809 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
810 	sr->len = READ_ONCE(sqe->len);
811 	if (unlikely(sr->len < 0))
812 		return -EINVAL;
813 	sr->flags = READ_ONCE(sqe->ioprio);
814 	if (sr->flags & ~RECVMSG_FLAGS)
815 		return -EINVAL;
816 	if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
817 		/* registered buffer recv only for plain IORING_OP_RECV */
818 		if (req->opcode != IORING_OP_RECV ||
819 		    (req->flags & REQ_F_BUFFER_SELECT) ||
820 		    (sr->flags & (IORING_RECV_MULTISHOT | IORING_RECVSEND_BUNDLE)))
821 			return -EINVAL;
822 		req->buf_index = READ_ONCE(sqe->buf_index);
823 	}
824 	sr->msg_flags = READ_ONCE(sqe->msg_flags);
825 	if (sr->msg_flags & MSG_DONTWAIT)
826 		req->flags |= REQ_F_NOWAIT;
827 	if (sr->msg_flags & MSG_ERRQUEUE)
828 		req->flags |= REQ_F_CLEAR_POLLIN;
829 	if (req->flags & REQ_F_BUFFER_SELECT)
830 		sr->buf_group = req->buf_index;
831 	sr->mshot_total_len = sr->mshot_len = 0;
832 	if (sr->flags & IORING_RECV_MULTISHOT) {
833 		if (!(req->flags & REQ_F_BUFFER_SELECT))
834 			return -EINVAL;
835 		if (sr->msg_flags & MSG_WAITALL)
836 			return -EINVAL;
837 		if (req->opcode == IORING_OP_RECV) {
838 			sr->mshot_len = sr->len;
839 			sr->mshot_total_len = READ_ONCE(sqe->optlen);
840 			if (sr->mshot_total_len)
841 				sr->flags |= IORING_RECV_MSHOT_LIM;
842 		} else if (sqe->optlen) {
843 			return -EINVAL;
844 		}
845 		req->flags |= REQ_F_APOLL_MULTISHOT;
846 	} else if (sqe->optlen) {
847 		return -EINVAL;
848 	}
849 
850 	if (sr->flags & IORING_RECVSEND_BUNDLE) {
851 		if (req->opcode == IORING_OP_RECVMSG)
852 			return -EINVAL;
853 	}
854 
855 	if (io_is_compat(req->ctx))
856 		sr->msg_flags |= MSG_CMSG_COMPAT;
857 
858 	sr->nr_multishot_loops = 0;
859 	return io_recvmsg_prep_setup(req);
860 }
861 
862 /* bits to clear in old and inherit in new cflags on bundle retry */
863 #define CQE_F_MASK	(IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE|\
864 			 IORING_CQE_F_BUF_MORE)
865 
866 /*
867  * Finishes io_recv and io_recvmsg.
868  *
869  * Returns true if it is actually finished, or false if it should run
870  * again (for multishot).
871  */
872 static inline bool io_recv_finish(struct io_kiocb *req,
873 				  struct io_async_msghdr *kmsg,
874 				  struct io_br_sel *sel, bool mshot_finished,
875 				  unsigned issue_flags, int consumed)
876 {
877 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
878 	unsigned int cflags = 0;
879 
880 	if (kmsg->msg.msg_inq > 0)
881 		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
882 
883 	if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) {
884 		unsigned int this_len = sel->val - sr->done_io;
885 
886 		/*
887 		 * If sr->mshot_total_len hits zero, the limit has been reached.
888 		 * Mark mshot as finished, and flag MSHOT_DONE as well to
889 		 * prevent a potential bundle from being retried.
890 		 */
891 		sr->mshot_total_len -= min_t(unsigned int, this_len,
892 					     sr->mshot_total_len);
893 		if (!sr->mshot_total_len) {
894 			sr->flags |= IORING_RECV_MSHOT_DONE;
895 			mshot_finished = true;
896 		}
897 	}
898 
899 	if (sr->flags & IORING_RECVSEND_BUNDLE) {
900 		size_t this_ret = sel->val - sr->done_io;
901 
902 		cflags |= io_put_kbufs(req, consumed, sel->buf_list, io_bundle_nbufs(kmsg, consumed));
903 		if (sr->flags & IORING_RECV_RETRY)
904 			cflags = req->cqe.flags | (cflags & CQE_F_MASK);
905 		if (sr->mshot_len && sel->val >= sr->mshot_len)
906 			sr->flags |= IORING_RECV_MSHOT_CAP;
907 		/* bundle with no more immediate buffers, we're done */
908 		if (req->flags & REQ_F_BL_EMPTY)
909 			goto finish;
910 		/*
911 		 * If more is available AND it was a full transfer, retry and
912 		 * append to this one
913 		 */
914 		if (!(sr->flags & IORING_RECV_NO_RETRY) &&
915 		    kmsg->msg.msg_inq > 1 && this_ret > 0 &&
916 		    !iov_iter_count(&kmsg->msg.msg_iter)) {
917 			req->cqe.flags = cflags & ~CQE_F_MASK;
918 			sr->len = kmsg->msg.msg_inq;
919 			sr->done_io += this_ret;
920 			sr->flags |= IORING_RECV_RETRY;
921 			return false;
922 		}
923 	} else {
924 		cflags |= io_put_kbuf(req, consumed, sel->buf_list);
925 	}
926 
927 	/*
928 	 * Fill CQE for this receive and see if we should keep trying to
929 	 * receive from this socket.
930 	 */
931 	if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&
932 	    io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
933 		sel->val = IOU_RETRY;
934 		io_mshot_prep_retry(req, kmsg);
935 		/* Known not-empty or unknown state, retry */
936 		if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {
937 			if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY &&
938 			    !(sr->flags & IORING_RECV_MSHOT_CAP)) {
939 				return false;
940 			}
941 			/* mshot retries exceeded, force a requeue */
942 			sr->nr_multishot_loops = 0;
943 			sr->flags &= ~IORING_RECV_MSHOT_CAP;
944 			if (issue_flags & IO_URING_F_MULTISHOT)
945 				sel->val = IOU_REQUEUE;
946 		}
947 		return true;
948 	}
949 
950 	/* Finish the request / stop multishot. */
951 finish:
952 	io_req_set_res(req, sel->val, cflags);
953 	sel->val = IOU_COMPLETE;
954 	io_req_msg_cleanup(req, issue_flags);
955 	return true;
956 }
957 
958 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,
959 				     struct io_sr_msg *sr, void __user **buf,
960 				     size_t *len)
961 {
962 	unsigned long ubuf = (unsigned long) *buf;
963 	unsigned long hdr;
964 
965 	hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
966 		kmsg->controllen;
967 	if (*len < hdr)
968 		return -EFAULT;
969 
970 	if (kmsg->controllen) {
971 		unsigned long control = ubuf + hdr - kmsg->controllen;
972 
973 		kmsg->msg.msg_control_user = (void __user *) control;
974 		kmsg->msg.msg_controllen = kmsg->controllen;
975 	}
976 
977 	sr->buf = *buf; /* stash for later copy */
978 	*buf = (void __user *) (ubuf + hdr);
979 	kmsg->payloadlen = *len = *len - hdr;
980 	return 0;
981 }
982 
983 struct io_recvmsg_multishot_hdr {
984 	struct io_uring_recvmsg_out msg;
985 	struct sockaddr_storage addr;
986 };
987 
988 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io,
989 				struct io_async_msghdr *kmsg,
990 				unsigned int flags, bool *finished)
991 {
992 	int err;
993 	int copy_len;
994 	struct io_recvmsg_multishot_hdr hdr;
995 
996 	if (kmsg->namelen)
997 		kmsg->msg.msg_name = &hdr.addr;
998 	kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);
999 	kmsg->msg.msg_namelen = 0;
1000 
1001 	if (sock->file->f_flags & O_NONBLOCK)
1002 		flags |= MSG_DONTWAIT;
1003 
1004 	err = sock_recvmsg(sock, &kmsg->msg, flags);
1005 	*finished = err <= 0;
1006 	if (err < 0)
1007 		return err;
1008 
1009 	hdr.msg = (struct io_uring_recvmsg_out) {
1010 		.controllen = kmsg->controllen - kmsg->msg.msg_controllen,
1011 		.flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT
1012 	};
1013 
1014 	hdr.msg.payloadlen = err;
1015 	if (err > kmsg->payloadlen)
1016 		err = kmsg->payloadlen;
1017 
1018 	copy_len = sizeof(struct io_uring_recvmsg_out);
1019 	if (kmsg->msg.msg_namelen > kmsg->namelen)
1020 		copy_len += kmsg->namelen;
1021 	else
1022 		copy_len += kmsg->msg.msg_namelen;
1023 
1024 	/*
1025 	 *      "fromlen shall refer to the value before truncation.."
1026 	 *                      1003.1g
1027 	 */
1028 	hdr.msg.namelen = kmsg->msg.msg_namelen;
1029 
1030 	/* ensure that there is no gap between hdr and sockaddr_storage */
1031 	BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) !=
1032 		     sizeof(struct io_uring_recvmsg_out));
1033 	if (copy_to_user(io->buf, &hdr, copy_len)) {
1034 		*finished = true;
1035 		return -EFAULT;
1036 	}
1037 
1038 	return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
1039 			kmsg->controllen + err;
1040 }
1041 
1042 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
1043 {
1044 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1045 	struct io_async_msghdr *kmsg = req->async_data;
1046 	struct io_br_sel sel = { };
1047 	struct socket *sock;
1048 	unsigned flags;
1049 	int ret, min_ret = 0;
1050 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1051 	bool mshot_finished = true;
1052 	int consumed = 0;
1053 	size_t len;
1054 
1055 	sock = sock_from_file(req->file);
1056 	if (unlikely(!sock))
1057 		return -ENOTSOCK;
1058 
1059 	if (!(req->flags & REQ_F_POLLED) &&
1060 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
1061 		return -EAGAIN;
1062 
1063 	flags = sr->msg_flags;
1064 	if (force_nonblock)
1065 		flags |= MSG_DONTWAIT;
1066 
1067 retry_multishot:
1068 	sel.buf_list = NULL;
1069 	len = sr->len;
1070 	if (io_do_buffer_select(req)) {
1071 		sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1072 		if (!sel.addr)
1073 			return -ENOBUFS;
1074 		len = min_t(size_t, len, MAX_RW_COUNT);
1075 
1076 		if (req->flags & REQ_F_APOLL_MULTISHOT) {
1077 			ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
1078 			if (ret) {
1079 				io_kbuf_recycle(req, sel.buf_list, issue_flags);
1080 				return ret;
1081 			}
1082 		}
1083 
1084 		iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
1085 	}
1086 
1087 	kmsg->msg.msg_get_inq = 1;
1088 	kmsg->msg.msg_inq = -1;
1089 	if (req->flags & REQ_F_APOLL_MULTISHOT) {
1090 		ret = io_recvmsg_multishot(sock, sr, kmsg, flags,
1091 					   &mshot_finished);
1092 		consumed = ret;
1093 	} else {
1094 		/* disable partial retry for recvmsg with cmsg attached */
1095 		if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen)
1096 			min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1097 
1098 		ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,
1099 					 kmsg->uaddr, flags);
1100 		/*
1101 		 * With MSG_TRUNC, the net layer will return the full size of
1102 		 * the packet, even if we only filled part of it in the buffers.
1103 		 * Adjust the returned size to consume only the real part of the
1104 		 * buffer.
1105 		 */
1106 		consumed = ret;
1107 		if (ret > 0)
1108 			consumed = min_t(size_t, ret, len);
1109 	}
1110 
1111 	if (ret < min_ret) {
1112 		if (ret == -EAGAIN && force_nonblock) {
1113 			io_kbuf_recycle(req, sel.buf_list, issue_flags);
1114 			return IOU_RETRY;
1115 		}
1116 		if (ret > 0 && io_net_retry(sock, flags) &&
1117 		    io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) {
1118 			sr->done_io += ret;
1119 			return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
1120 		}
1121 		if (ret == -ERESTARTSYS)
1122 			ret = -EINTR;
1123 		req_set_fail(req);
1124 	} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1125 		req_set_fail(req);
1126 	}
1127 
1128 	if (ret > 0)
1129 		ret += sr->done_io;
1130 	else if (sr->done_io)
1131 		ret = sr->done_io;
1132 	else
1133 		io_kbuf_recycle(req, sel.buf_list, issue_flags);
1134 
1135 	sel.val = ret;
1136 	if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed))
1137 		goto retry_multishot;
1138 
1139 	return sel.val;
1140 }
1141 
1142 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,
1143 			      struct io_br_sel *sel, unsigned int issue_flags)
1144 {
1145 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1146 	size_t len;
1147 	int ret;
1148 
1149 	/*
1150 	 * If the ring isn't locked, then don't use the peek interface
1151 	 * to grab multiple buffers as we will lock/unlock between
1152 	 * this selection and posting the buffers.
1153 	 */
1154 	if (!(issue_flags & IO_URING_F_UNLOCKED) &&
1155 	    sr->flags & IORING_RECVSEND_BUNDLE) {
1156 		struct buf_sel_arg arg = {
1157 			.iovs = &kmsg->fast_iov,
1158 			.nr_iovs = 1,
1159 			.mode = KBUF_MODE_EXPAND,
1160 			.buf_group = sr->buf_group,
1161 		};
1162 
1163 		if (kmsg->vec.iovec) {
1164 			arg.nr_iovs = kmsg->vec.nr;
1165 			arg.iovs = kmsg->vec.iovec;
1166 			arg.mode |= KBUF_MODE_FREE;
1167 		}
1168 
1169 		if (sel->val)
1170 			arg.max_len = sel->val;
1171 		else if (kmsg->msg.msg_inq > 1)
1172 			arg.max_len = kmsg->msg.msg_inq;
1173 
1174 		/* if mshot limited, ensure we don't go over */
1175 		if (sr->flags & IORING_RECV_MSHOT_LIM)
1176 			arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len);
1177 		ret = io_buffers_peek(req, &arg, sel);
1178 		if (unlikely(ret < 0))
1179 			return ret;
1180 
1181 		if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
1182 			kmsg->vec.nr = ret;
1183 			kmsg->vec.iovec = arg.iovs;
1184 			req->flags |= REQ_F_NEED_CLEANUP;
1185 		}
1186 		if (arg.partial_map)
1187 			sr->flags |= IORING_RECV_PARTIAL_MAP;
1188 
1189 		/* special case 1 vec, can be a fast path */
1190 		if (ret == 1) {
1191 			sr->buf = arg.iovs[0].iov_base;
1192 			len = sr->len = arg.iovs[0].iov_len;
1193 			goto map_ubuf;
1194 		}
1195 		iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
1196 			      arg.out_len);
1197 		len = arg.out_len;
1198 	} else {
1199 		len = sel->val;
1200 
1201 		*sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1202 		if (!sel->addr)
1203 			return -ENOBUFS;
1204 		/* ring provided buffers can be up to 4G, keep len an int */
1205 		len = min_t(size_t, len, MAX_RW_COUNT);
1206 		sr->buf = sel->addr;
1207 		sr->len = len;
1208 map_ubuf:
1209 		ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
1210 				  &kmsg->msg.msg_iter);
1211 		if (unlikely(ret))
1212 			return ret;
1213 	}
1214 
1215 	return len;
1216 }
1217 
1218 int io_recv(struct io_kiocb *req, unsigned int issue_flags)
1219 {
1220 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1221 	struct io_async_msghdr *kmsg = req->async_data;
1222 	struct io_br_sel sel;
1223 	struct socket *sock;
1224 	unsigned flags;
1225 	int ret, min_ret = 0, consumed = 0;
1226 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1227 	bool mshot_finished;
1228 	size_t len = 0;
1229 
1230 	sock = sock_from_file(req->file);
1231 	if (unlikely(!sock))
1232 		return -ENOTSOCK;
1233 
1234 	if (!(req->flags & REQ_F_POLLED) &&
1235 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
1236 		return -EAGAIN;
1237 
1238 	flags = sr->msg_flags;
1239 	if (force_nonblock)
1240 		flags |= MSG_DONTWAIT;
1241 
1242 	if (req->flags & REQ_F_IMPORT_BUFFER) {
1243 		ret = io_import_reg_buf(req, &kmsg->msg.msg_iter,
1244 					(u64)(uintptr_t)sr->buf, sr->len,
1245 					ITER_DEST, issue_flags);
1246 		if (unlikely(ret)) {
1247 			kmsg->msg.msg_inq = -1;
1248 			sel.buf_list = NULL;
1249 			goto out_free;
1250 		}
1251 		req->flags &= ~REQ_F_IMPORT_BUFFER;
1252 	}
1253 
1254 retry_multishot:
1255 	sel.buf_list = NULL;
1256 	len = sr->len;
1257 	if (io_do_buffer_select(req)) {
1258 		sel.val = sr->len;
1259 		ret = io_recv_buf_select(req, kmsg, &sel, issue_flags);
1260 		if (unlikely(ret < 0)) {
1261 			kmsg->msg.msg_inq = -1;
1262 			goto out_free;
1263 		}
1264 		len = ret;
1265 		sr->buf = NULL;
1266 	}
1267 
1268 	kmsg->msg.msg_flags = 0;
1269 	kmsg->msg.msg_inq = -1;
1270 
1271 	if (flags & MSG_WAITALL)
1272 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1273 
1274 	ret = sock_recvmsg(sock, &kmsg->msg, flags);
1275 	if (ret < min_ret) {
1276 		if (ret == -EAGAIN && force_nonblock) {
1277 			io_kbuf_recycle(req, sel.buf_list, issue_flags);
1278 			return IOU_RETRY;
1279 		}
1280 		if (ret > 0 && io_net_retry(sock, flags) &&
1281 		    io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) {
1282 			sr->len -= ret;
1283 			sr->buf += ret;
1284 			sr->done_io += ret;
1285 			return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret);
1286 		}
1287 		if (ret == -ERESTARTSYS)
1288 			ret = -EINTR;
1289 		req_set_fail(req);
1290 	} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1291 out_free:
1292 		req_set_fail(req);
1293 	}
1294 
1295 	mshot_finished = ret <= 0;
1296 
1297 	/*
1298 	 * With MSG_TRUNC, the net layer will return the full size of
1299 	 * the packet, even if we only filled part of it in the buffers.
1300 	 * Adjust the returned size to consume only the real part of the
1301 	 * buffer.
1302 	 */
1303 	consumed = ret;
1304 	if (ret > 0)
1305 		consumed = min_t(size_t, ret, len);
1306 
1307 	if (ret > 0)
1308 		ret += sr->done_io;
1309 	else if (sr->done_io)
1310 		ret = sr->done_io;
1311 	else
1312 		io_kbuf_recycle(req, sel.buf_list, issue_flags);
1313 
1314 	sel.val = ret;
1315 	if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed))
1316 		goto retry_multishot;
1317 
1318 	return sel.val;
1319 }
1320 
1321 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1322 {
1323 	struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);
1324 	unsigned ifq_idx;
1325 
1326 	if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3))
1327 		return -EINVAL;
1328 
1329 	ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx);
1330 	zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx);
1331 	if (!zc->ifq)
1332 		return -EINVAL;
1333 
1334 	zc->len = READ_ONCE(sqe->len);
1335 	zc->flags = READ_ONCE(sqe->ioprio);
1336 	if (READ_ONCE(sqe->msg_flags))
1337 		return -EINVAL;
1338 	if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT))
1339 		return -EINVAL;
1340 	/* multishot required */
1341 	if (!(zc->flags & IORING_RECV_MULTISHOT))
1342 		return -EINVAL;
1343 	/* All data completions are posted as aux CQEs. */
1344 	req->flags |= REQ_F_APOLL_MULTISHOT;
1345 
1346 	return 0;
1347 }
1348 
1349 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags)
1350 {
1351 	struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);
1352 	struct socket *sock;
1353 	unsigned int len;
1354 	int ret;
1355 
1356 	sock = sock_from_file(req->file);
1357 	if (unlikely(!sock))
1358 		return -ENOTSOCK;
1359 
1360 	if (!(req->flags & REQ_F_POLLED) &&
1361 	    (zc->flags & IORING_RECVSEND_POLL_FIRST))
1362 		return -EAGAIN;
1363 
1364 	len = zc->len;
1365 	ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len);
1366 	if (len && zc->len == 0) {
1367 		io_req_set_res(req, 0, 0);
1368 
1369 		return IOU_COMPLETE;
1370 	}
1371 	if (unlikely(ret <= 0) && ret != -EAGAIN) {
1372 		if (ret == -ERESTARTSYS)
1373 			ret = -EINTR;
1374 		if (ret == IOU_REQUEUE)
1375 			return IOU_REQUEUE;
1376 
1377 		req_set_fail(req);
1378 		io_req_set_res(req, ret, 0);
1379 		return IOU_COMPLETE;
1380 	}
1381 	return IOU_RETRY;
1382 }
1383 
1384 void io_send_zc_cleanup(struct io_kiocb *req)
1385 {
1386 	struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1387 	struct io_async_msghdr *io = req->async_data;
1388 
1389 	if (req_has_async_data(req))
1390 		io_netmsg_iovec_free(io);
1391 	if (zc->notif) {
1392 		io_notif_flush(zc->notif);
1393 		zc->notif = NULL;
1394 	}
1395 }
1396 
1397 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF)
1398 #define IO_ZC_FLAGS_VALID  (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \
1399 				IORING_SEND_VECTORIZED)
1400 
1401 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1402 {
1403 	struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1404 	struct io_ring_ctx *ctx = req->ctx;
1405 	struct io_async_msghdr *iomsg;
1406 	struct io_kiocb *notif;
1407 	u64 user_data;
1408 	int ret;
1409 
1410 	zc->done_io = 0;
1411 
1412 	if (unlikely(READ_ONCE(sqe->__pad2[0])))
1413 		return -EINVAL;
1414 	/* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
1415 	if (req->flags & REQ_F_CQE_SKIP)
1416 		return -EINVAL;
1417 
1418 	notif = zc->notif = io_alloc_notif(ctx);
1419 	if (!notif)
1420 		return -ENOMEM;
1421 	user_data = READ_ONCE(sqe->addr3);
1422 	if (!user_data)
1423 		user_data = req->cqe.user_data;
1424 
1425 	notif->cqe.user_data = user_data;
1426 	notif->cqe.res = 0;
1427 	notif->cqe.flags = IORING_CQE_F_NOTIF;
1428 	req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY;
1429 
1430 	zc->flags = READ_ONCE(sqe->ioprio);
1431 	if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) {
1432 		if (zc->flags & ~IO_ZC_FLAGS_VALID)
1433 			return -EINVAL;
1434 		if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) {
1435 			struct io_notif_data *nd = io_notif_to_data(notif);
1436 
1437 			nd->zc_report = true;
1438 			nd->zc_used = false;
1439 			nd->zc_copied = false;
1440 		}
1441 	}
1442 
1443 	zc->len = READ_ONCE(sqe->len);
1444 	zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY;
1445 	req->buf_index = READ_ONCE(sqe->buf_index);
1446 	if (zc->msg_flags & MSG_DONTWAIT)
1447 		req->flags |= REQ_F_NOWAIT;
1448 
1449 	if (io_is_compat(ctx))
1450 		zc->msg_flags |= MSG_CMSG_COMPAT;
1451 
1452 	iomsg = io_msg_alloc_async(req);
1453 	if (unlikely(!iomsg))
1454 		return -ENOMEM;
1455 
1456 	if (req->opcode == IORING_OP_SEND_ZC) {
1457 		ret = io_send_setup(req, sqe);
1458 	} else {
1459 		if (unlikely(sqe->addr2 || sqe->file_index))
1460 			return -EINVAL;
1461 		ret = io_sendmsg_setup(req, sqe);
1462 	}
1463 	if (unlikely(ret))
1464 		return ret;
1465 
1466 	if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) {
1467 		iomsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1468 		return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count);
1469 	}
1470 	iomsg->msg.sg_from_iter = io_sg_from_iter;
1471 	return 0;
1472 }
1473 
1474 static int io_sg_from_iter_iovec(struct sk_buff *skb,
1475 				 struct iov_iter *from, size_t length)
1476 {
1477 	skb_zcopy_downgrade_managed(skb);
1478 	return zerocopy_fill_skb_from_iter(skb, from, length);
1479 }
1480 
1481 static int io_sg_from_iter(struct sk_buff *skb,
1482 			   struct iov_iter *from, size_t length)
1483 {
1484 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1485 	int frag = shinfo->nr_frags;
1486 	int ret = 0;
1487 	struct bvec_iter bi;
1488 	ssize_t copied = 0;
1489 	unsigned long truesize = 0;
1490 
1491 	if (!frag)
1492 		shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
1493 	else if (unlikely(!skb_zcopy_managed(skb)))
1494 		return zerocopy_fill_skb_from_iter(skb, from, length);
1495 
1496 	bi.bi_size = min(from->count, length);
1497 	bi.bi_offset = from->iov_offset;
1498 	bi.bi_idx = 0;
1499 
1500 	while (bi.bi_size && frag < MAX_SKB_FRAGS) {
1501 		struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi);
1502 
1503 		copied += v.bv_len;
1504 		truesize += PAGE_ALIGN(v.bv_len + v.bv_offset);
1505 		__skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page,
1506 					   v.bv_offset, v.bv_len);
1507 		bvec_iter_advance_single(from->bvec, &bi, v.bv_len);
1508 	}
1509 	if (bi.bi_size)
1510 		ret = -EMSGSIZE;
1511 
1512 	shinfo->nr_frags = frag;
1513 	from->bvec += bi.bi_idx;
1514 	from->nr_segs -= bi.bi_idx;
1515 	from->count -= copied;
1516 	from->iov_offset = bi.bi_offset;
1517 
1518 	skb->data_len += copied;
1519 	skb->len += copied;
1520 	skb->truesize += truesize;
1521 	return ret;
1522 }
1523 
1524 static int io_send_zc_import(struct io_kiocb *req,
1525 			     struct io_async_msghdr *kmsg,
1526 			     unsigned int issue_flags)
1527 {
1528 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1529 	struct io_kiocb *notif = sr->notif;
1530 	int ret;
1531 
1532 	WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF));
1533 
1534 	notif->buf_index = req->buf_index;
1535 
1536 	if (!(sr->flags & IORING_SEND_VECTORIZED)) {
1537 		ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter,
1538 					(u64)(uintptr_t)sr->buf, sr->len,
1539 					ITER_SOURCE, issue_flags);
1540 	} else {
1541 		unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;
1542 
1543 		ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter,
1544 					notif, &kmsg->vec, uvec_segs,
1545 					issue_flags);
1546 	}
1547 
1548 	if (unlikely(ret))
1549 		return ret;
1550 	req->flags &= ~REQ_F_IMPORT_BUFFER;
1551 	return 0;
1552 }
1553 
1554 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
1555 {
1556 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1557 	struct io_async_msghdr *kmsg = req->async_data;
1558 	struct socket *sock;
1559 	unsigned msg_flags;
1560 	int ret, min_ret = 0;
1561 
1562 	sock = sock_from_file(req->file);
1563 	if (unlikely(!sock))
1564 		return -ENOTSOCK;
1565 	if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1566 		return -EOPNOTSUPP;
1567 	if (!(req->flags & REQ_F_POLLED) &&
1568 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
1569 		return -EAGAIN;
1570 
1571 	if (req->flags & REQ_F_IMPORT_BUFFER) {
1572 		ret = io_send_zc_import(req, kmsg, issue_flags);
1573 		if (unlikely(ret))
1574 			return ret;
1575 	}
1576 
1577 	msg_flags = sr->msg_flags;
1578 	if (issue_flags & IO_URING_F_NONBLOCK)
1579 		msg_flags |= MSG_DONTWAIT;
1580 	if (msg_flags & MSG_WAITALL)
1581 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1582 
1583 	kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1584 
1585 	if (req->opcode == IORING_OP_SEND_ZC) {
1586 		msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
1587 		kmsg->msg.msg_flags = msg_flags;
1588 		ret = sock_sendmsg(sock, &kmsg->msg);
1589 	} else {
1590 		kmsg->msg.msg_control_user = sr->msg_control;
1591 		ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags);
1592 	}
1593 
1594 	if (unlikely(ret < min_ret)) {
1595 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1596 			return -EAGAIN;
1597 
1598 		if (ret > 0 && io_net_retry(sock, sr->msg_flags)) {
1599 			sr->done_io += ret;
1600 			return -EAGAIN;
1601 		}
1602 		if (ret == -ERESTARTSYS)
1603 			ret = -EINTR;
1604 		req_set_fail(req);
1605 	}
1606 
1607 	if (ret >= 0)
1608 		ret += sr->done_io;
1609 	else if (sr->done_io)
1610 		ret = sr->done_io;
1611 
1612 	/*
1613 	 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1614 	 * flushing notif to io_send_zc_cleanup()
1615 	 */
1616 	if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1617 		io_notif_flush(sr->notif);
1618 		sr->notif = NULL;
1619 		io_req_msg_cleanup(req, 0);
1620 	}
1621 	io_req_set_res(req, ret, IORING_CQE_F_MORE);
1622 	return IOU_COMPLETE;
1623 }
1624 
1625 void io_sendrecv_fail(struct io_kiocb *req)
1626 {
1627 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1628 
1629 	if (sr->done_io)
1630 		req->cqe.res = sr->done_io;
1631 
1632 	if ((req->flags & REQ_F_NEED_CLEANUP) &&
1633 	    (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC))
1634 		req->cqe.flags |= IORING_CQE_F_MORE;
1635 }
1636 
1637 #define ACCEPT_FLAGS	(IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \
1638 			 IORING_ACCEPT_POLL_FIRST)
1639 
1640 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1641 {
1642 	struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1643 
1644 	if (sqe->len || sqe->buf_index)
1645 		return -EINVAL;
1646 
1647 	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1648 	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
1649 	accept->flags = READ_ONCE(sqe->accept_flags);
1650 	accept->nofile = rlimit(RLIMIT_NOFILE);
1651 	accept->iou_flags = READ_ONCE(sqe->ioprio);
1652 	if (accept->iou_flags & ~ACCEPT_FLAGS)
1653 		return -EINVAL;
1654 
1655 	accept->file_slot = READ_ONCE(sqe->file_index);
1656 	if (accept->file_slot) {
1657 		if (accept->flags & SOCK_CLOEXEC)
1658 			return -EINVAL;
1659 		if (accept->iou_flags & IORING_ACCEPT_MULTISHOT &&
1660 		    accept->file_slot != IORING_FILE_INDEX_ALLOC)
1661 			return -EINVAL;
1662 	}
1663 	if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1664 		return -EINVAL;
1665 	if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
1666 		accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
1667 	if (accept->iou_flags & IORING_ACCEPT_MULTISHOT)
1668 		req->flags |= REQ_F_APOLL_MULTISHOT;
1669 	if (accept->iou_flags & IORING_ACCEPT_DONTWAIT)
1670 		req->flags |= REQ_F_NOWAIT;
1671 	return 0;
1672 }
1673 
1674 int io_accept(struct io_kiocb *req, unsigned int issue_flags)
1675 {
1676 	struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
1677 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1678 	bool fixed = !!accept->file_slot;
1679 	struct proto_accept_arg arg = {
1680 		.flags = force_nonblock ? O_NONBLOCK : 0,
1681 	};
1682 	struct file *file;
1683 	unsigned cflags;
1684 	int ret, fd;
1685 
1686 	if (!(req->flags & REQ_F_POLLED) &&
1687 	    accept->iou_flags & IORING_ACCEPT_POLL_FIRST)
1688 		return -EAGAIN;
1689 
1690 retry:
1691 	if (!fixed) {
1692 		fd = __get_unused_fd_flags(accept->flags, accept->nofile);
1693 		if (unlikely(fd < 0))
1694 			return fd;
1695 	}
1696 	arg.err = 0;
1697 	arg.is_empty = -1;
1698 	file = do_accept(req->file, &arg, accept->addr, accept->addr_len,
1699 			 accept->flags);
1700 	if (IS_ERR(file)) {
1701 		if (!fixed)
1702 			put_unused_fd(fd);
1703 		ret = PTR_ERR(file);
1704 		if (ret == -EAGAIN && force_nonblock &&
1705 		    !(accept->iou_flags & IORING_ACCEPT_DONTWAIT))
1706 			return IOU_RETRY;
1707 
1708 		if (ret == -ERESTARTSYS)
1709 			ret = -EINTR;
1710 	} else if (!fixed) {
1711 		fd_install(fd, file);
1712 		ret = fd;
1713 	} else {
1714 		ret = io_fixed_fd_install(req, issue_flags, file,
1715 						accept->file_slot);
1716 	}
1717 
1718 	cflags = 0;
1719 	if (!arg.is_empty)
1720 		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
1721 
1722 	if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) &&
1723 	    io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
1724 		if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1)
1725 			goto retry;
1726 		return IOU_RETRY;
1727 	}
1728 
1729 	io_req_set_res(req, ret, cflags);
1730 	if (ret < 0)
1731 		req_set_fail(req);
1732 	return IOU_COMPLETE;
1733 }
1734 
1735 void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
1736 {
1737 	struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1738 
1739 	bctx->socket.family = sock->domain;
1740 	bctx->socket.type = sock->type & SOCK_TYPE_MASK;
1741 	bctx->socket.protocol = sock->protocol;
1742 }
1743 
1744 void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
1745 {
1746 	struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
1747 	struct sockaddr_storage *ss = req->async_data;
1748 
1749 	/*
1750 	 * move_addr_to_kernel() skips the copy for addr_len == 0, so
1751 	 * iomsg->addr may hold stale data from a prior CONNECT. Bail
1752 	 * unless addr_len covers the family discriminator.
1753 	 */
1754 	if (conn->addr_len < (int)sizeof(sa_family_t))
1755 		return;
1756 
1757 	bctx->connect.family = ss->ss_family;
1758 	switch (ss->ss_family) {
1759 	case AF_INET: {
1760 		struct sockaddr_in *sin = (struct sockaddr_in *)ss;
1761 
1762 		if (conn->addr_len < (int)sizeof(*sin))
1763 			break;
1764 		bctx->connect.port = sin->sin_port;
1765 		bctx->connect.v4_addr = sin->sin_addr.s_addr;
1766 		break;
1767 	}
1768 	case AF_INET6: {
1769 		struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;
1770 
1771 		/* The IPv6 stack accepts SIN6_LEN_RFC2133; port and addr are in it. */
1772 		if (conn->addr_len < SIN6_LEN_RFC2133)
1773 			break;
1774 		bctx->connect.port = sin6->sin6_port;
1775 		memcpy(bctx->connect.v6_addr, &sin6->sin6_addr,
1776 		       sizeof(bctx->connect.v6_addr));
1777 		break;
1778 	}
1779 	default:
1780 		/* family is set; per-family fields stay zero - family-only filtering */
1781 		break;
1782 	}
1783 }
1784 
1785 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1786 {
1787 	struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1788 
1789 	if (sqe->addr || sqe->rw_flags || sqe->buf_index)
1790 		return -EINVAL;
1791 
1792 	sock->domain = READ_ONCE(sqe->fd);
1793 	sock->type = READ_ONCE(sqe->off);
1794 	sock->protocol = READ_ONCE(sqe->len);
1795 	sock->file_slot = READ_ONCE(sqe->file_index);
1796 	sock->nofile = rlimit(RLIMIT_NOFILE);
1797 
1798 	sock->flags = sock->type & ~SOCK_TYPE_MASK;
1799 	if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
1800 		return -EINVAL;
1801 	if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1802 		return -EINVAL;
1803 	return 0;
1804 }
1805 
1806 int io_socket(struct io_kiocb *req, unsigned int issue_flags)
1807 {
1808 	struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
1809 	bool fixed = !!sock->file_slot;
1810 	struct file *file;
1811 	int ret, fd;
1812 
1813 	if (!fixed) {
1814 		fd = __get_unused_fd_flags(sock->flags, sock->nofile);
1815 		if (unlikely(fd < 0))
1816 			return fd;
1817 	}
1818 	file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
1819 	if (IS_ERR(file)) {
1820 		if (!fixed)
1821 			put_unused_fd(fd);
1822 		ret = PTR_ERR(file);
1823 		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1824 			return -EAGAIN;
1825 		if (ret == -ERESTARTSYS)
1826 			ret = -EINTR;
1827 		req_set_fail(req);
1828 	} else if (!fixed) {
1829 		fd_install(fd, file);
1830 		ret = fd;
1831 	} else {
1832 		ret = io_fixed_fd_install(req, issue_flags, file,
1833 					    sock->file_slot);
1834 	}
1835 	io_req_set_res(req, ret, 0);
1836 	return IOU_COMPLETE;
1837 }
1838 
1839 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1840 {
1841 	struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
1842 	struct sockaddr_storage *addr;
1843 
1844 	if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1845 		return -EINVAL;
1846 
1847 	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1848 	conn->addr_len =  READ_ONCE(sqe->addr2);
1849 	conn->in_progress = conn->seen_econnaborted = false;
1850 
1851 	addr = io_uring_alloc_async_data(NULL, req);
1852 	if (unlikely(!addr))
1853 		return -ENOMEM;
1854 
1855 	return move_addr_to_kernel(conn->addr, conn->addr_len, addr);
1856 }
1857 
1858 int io_connect(struct io_kiocb *req, unsigned int issue_flags)
1859 {
1860 	struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect);
1861 	struct sockaddr_storage *addr = req->async_data;
1862 	unsigned file_flags;
1863 	int ret;
1864 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
1865 
1866 	if (connect->in_progress) {
1867 		struct poll_table_struct pt = { ._key = EPOLLERR };
1868 
1869 		if (vfs_poll(req->file, &pt) & EPOLLERR)
1870 			goto get_sock_err;
1871 	}
1872 
1873 	file_flags = force_nonblock ? O_NONBLOCK : 0;
1874 
1875 	ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags);
1876 	if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED)
1877 	    && force_nonblock) {
1878 		if (ret == -EINPROGRESS) {
1879 			connect->in_progress = true;
1880 		} else if (ret == -ECONNABORTED) {
1881 			if (connect->seen_econnaborted)
1882 				goto out;
1883 			connect->seen_econnaborted = true;
1884 		} else if (req->file->f_flags & O_NONBLOCK) {
1885 			/*
1886 			 * __sys_connect_file() adds O_NONBLOCK from the socket
1887 			 * to every attempt, so a retry from io-wq fails with
1888 			 * -EAGAIN as well. A poll on the socket doesn't wait
1889 			 * until a retry can succeed. For example, an AF_UNIX
1890 			 * client reports EPOLLOUT right away while the
1891 			 * listener's backlog is full. Complete the request
1892 			 * with -EAGAIN, like connect(2) does.
1893 			 */
1894 			goto out;
1895 		}
1896 		return -EAGAIN;
1897 	}
1898 	if (connect->in_progress) {
1899 		/*
1900 		 * At least bluetooth will return -EBADFD on a re-connect
1901 		 * attempt, and it's (supposedly) also valid to get -EISCONN
1902 		 * which means the previous result is good. For both of these,
1903 		 * grab the sock_error() and use that for the completion.
1904 		 */
1905 		if (ret == -EBADFD || ret == -EISCONN) {
1906 get_sock_err:
1907 			ret = sock_error(sock_from_file(req->file)->sk);
1908 		}
1909 	}
1910 	if (ret == -ERESTARTSYS)
1911 		ret = -EINTR;
1912 out:
1913 	if (ret < 0)
1914 		req_set_fail(req);
1915 	io_req_set_res(req, ret, 0);
1916 	return IOU_COMPLETE;
1917 }
1918 
1919 /*
1920  * Check if bind request would potentially end up with filename_create(),
1921  * which in turn end up in mnt_want_write() which will grab the fs
1922  * percpu start write sem. This can trigger a lockdep warning.
1923  */
1924 static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len)
1925 {
1926 	const struct sockaddr_un *sun;
1927 
1928 	if (addr->ss_family != AF_UNIX)
1929 		return 0;
1930 	if (addr_len <= offsetof(struct sockaddr_un, sun_path))
1931 		return 0;
1932 	sun = (const struct sockaddr_un *) addr;
1933 	return sun->sun_path[0] != '\0';
1934 }
1935 
1936 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1937 {
1938 	struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1939 	struct sockaddr __user *uaddr;
1940 	struct sockaddr_storage *addr;
1941 	int ret;
1942 
1943 	if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1944 		return -EINVAL;
1945 
1946 	uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1947 	bind->addr_len =  READ_ONCE(sqe->addr2);
1948 
1949 	addr = io_uring_alloc_async_data(NULL, req);
1950 	if (unlikely(!addr))
1951 		return -ENOMEM;
1952 	ret = move_addr_to_kernel(uaddr, bind->addr_len, addr);
1953 	if (unlikely(ret))
1954 		return ret;
1955 	if (io_bind_file_create(addr, bind->addr_len))
1956 		req->flags |= REQ_F_FORCE_ASYNC;
1957 	return 0;
1958 }
1959 
1960 
1961 int io_bind(struct io_kiocb *req, unsigned int issue_flags)
1962 {
1963 	struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
1964 	struct sockaddr_storage *addr = req->async_data;
1965 	struct socket *sock;
1966 	int ret;
1967 
1968 	sock = sock_from_file(req->file);
1969 	if (unlikely(!sock))
1970 		return -ENOTSOCK;
1971 
1972 	ret = __sys_bind_socket(sock, addr, bind->addr_len);
1973 	if (ret < 0)
1974 		req_set_fail(req);
1975 	io_req_set_res(req, ret, 0);
1976 	return 0;
1977 }
1978 
1979 int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1980 {
1981 	struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1982 
1983 	if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2)
1984 		return -EINVAL;
1985 
1986 	listen->backlog = READ_ONCE(sqe->len);
1987 	return 0;
1988 }
1989 
1990 int io_listen(struct io_kiocb *req, unsigned int issue_flags)
1991 {
1992 	struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen);
1993 	struct socket *sock;
1994 	int ret;
1995 
1996 	sock = sock_from_file(req->file);
1997 	if (unlikely(!sock))
1998 		return -ENOTSOCK;
1999 
2000 	ret = __sys_listen_socket(sock, listen->backlog);
2001 	if (ret < 0)
2002 		req_set_fail(req);
2003 	io_req_set_res(req, ret, 0);
2004 	return 0;
2005 }
2006 
2007 void io_netmsg_cache_free(const void *entry)
2008 {
2009 	struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry;
2010 
2011 	io_vec_free(&kmsg->vec);
2012 	kfree(kmsg);
2013 }
2014