xref: /linux/fs/splice.c (revision f0c29a42bf3609336349f632c23bebd01532e8c0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * "splice": joining two ropes together by interweaving their strands.
4  *
5  * This is the "extended pipe" functionality, where a pipe is used as
6  * an arbitrary in-memory buffer. Think of a pipe as a small kernel
7  * buffer that you can use to transfer data from one end to the other.
8  *
9  * The traditional unix read/write is extended with a "splice()" operation
10  * that transfers data buffers to or from a pipe buffer.
11  *
12  * Named by Larry McVoy, original implementation from Linus, extended by
13  * Jens to support splicing to files, network, direct splicing, etc and
14  * fixing lots of bugs.
15  *
16  * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
17  * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
18  * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
19  *
20  */
21 #include <linux/bvec.h>
22 #include <linux/fs.h>
23 #include <linux/file.h>
24 #include <linux/pagemap.h>
25 #include <linux/splice.h>
26 #include <linux/memcontrol.h>
27 #include <linux/mm_inline.h>
28 #include <linux/swap.h>
29 #include <linux/writeback.h>
30 #include <linux/export.h>
31 #include <linux/syscalls.h>
32 #include <linux/uio.h>
33 #include <linux/fsnotify.h>
34 #include <linux/security.h>
35 #include <linux/gfp.h>
36 #include <linux/net.h>
37 #include <linux/socket.h>
38 #include <linux/sched/signal.h>
39 
40 #include "internal.h"
41 
42 /*
43  * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to
44  * indicate they support non-blocking reads or writes, we must clear it
45  * here if set to avoid blocking other users of this pipe if splice is
46  * being done on it.
47  */
48 static noinline void pipe_clear_nowait(struct file *file)
49 {
50 	fmode_t fmode = READ_ONCE(file->f_mode);
51 
52 	do {
53 		if (!(fmode & FMODE_NOWAIT))
54 			break;
55 	} while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT));
56 }
57 
58 /*
59  * Check whether the contents of buf is OK to access. Since the content
60  * is a page cache page, IO may be in flight.
61  */
62 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
63 				       struct pipe_buffer *buf)
64 {
65 	struct folio *folio = page_folio(buf->page);
66 	int err;
67 
68 	if (!folio_test_uptodate(folio)) {
69 		folio_lock(folio);
70 
71 		/*
72 		 * Folio got truncated/unhashed. This will cause a 0-byte
73 		 * splice, if this is the first page.
74 		 */
75 		if (!folio->mapping) {
76 			err = -ENODATA;
77 			goto error;
78 		}
79 
80 		/*
81 		 * Uh oh, read-error from disk.
82 		 */
83 		if (!folio_test_uptodate(folio)) {
84 			err = -EIO;
85 			goto error;
86 		}
87 
88 		/* Folio is ok after all, we are done */
89 		folio_unlock(folio);
90 	}
91 
92 	return 0;
93 error:
94 	folio_unlock(folio);
95 	return err;
96 }
97 
98 const struct pipe_buf_operations page_cache_pipe_buf_ops = {
99 	.confirm	= page_cache_pipe_buf_confirm,
100 	.release	= generic_pipe_buf_release,
101 	.get		= generic_pipe_buf_get,
102 };
103 
104 static const struct pipe_buf_operations user_page_pipe_buf_ops = {
105 	.release	= generic_pipe_buf_release,
106 	.get		= generic_pipe_buf_get,
107 };
108 
109 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
110 {
111 	if (wq_has_sleeper(&pipe->rd_wait))
112 		wake_up_interruptible_poll(&pipe->rd_wait,
113 					   EPOLLIN | EPOLLRDNORM);
114 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
115 }
116 
117 /**
118  * splice_to_pipe - fill passed data into a pipe
119  * @pipe:	pipe to fill
120  * @spd:	data to fill
121  *
122  * Description:
123  *    @spd contains a map of pages and len/offset tuples, along with
124  *    the struct pipe_buf_operations associated with these pages. This
125  *    function will link that data to the pipe.
126  *
127  */
128 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
129 		       struct splice_pipe_desc *spd)
130 {
131 	unsigned int spd_pages = spd->nr_pages;
132 	unsigned int tail = pipe->tail;
133 	unsigned int head = pipe->head;
134 	ssize_t ret = 0;
135 	int page_nr = 0;
136 
137 	if (!spd_pages)
138 		return 0;
139 
140 	if (unlikely(!pipe->readers)) {
141 		send_sig(SIGPIPE, current, 0);
142 		ret = -EPIPE;
143 		goto out;
144 	}
145 
146 	while (!pipe_full(head, tail, pipe->max_usage)) {
147 		struct pipe_buffer *buf = pipe_buf(pipe, head);
148 
149 		buf->page = spd->pages[page_nr];
150 		buf->offset = spd->partial[page_nr].offset;
151 		buf->len = spd->partial[page_nr].len;
152 		buf->private = spd->partial[page_nr].private;
153 		buf->ops = spd->ops;
154 		buf->flags = 0;
155 
156 		head++;
157 		pipe->head = head;
158 		page_nr++;
159 		ret += buf->len;
160 
161 		if (!--spd->nr_pages)
162 			break;
163 	}
164 
165 	if (!ret)
166 		ret = -EAGAIN;
167 
168 out:
169 	while (page_nr < spd_pages)
170 		spd->spd_release(spd, page_nr++);
171 
172 	return ret;
173 }
174 EXPORT_SYMBOL_GPL(splice_to_pipe);
175 
176 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
177 {
178 	unsigned int head = pipe->head;
179 	unsigned int tail = pipe->tail;
180 	int ret;
181 
182 	if (unlikely(!pipe->readers)) {
183 		send_sig(SIGPIPE, current, 0);
184 		ret = -EPIPE;
185 	} else if (pipe_full(head, tail, pipe->max_usage)) {
186 		ret = -EAGAIN;
187 	} else {
188 		*pipe_buf(pipe, head) = *buf;
189 		pipe->head = head + 1;
190 		return buf->len;
191 	}
192 	pipe_buf_release(pipe, buf);
193 	return ret;
194 }
195 EXPORT_SYMBOL(add_to_pipe);
196 
197 /*
198  * Check if we need to grow the arrays holding pages and partial page
199  * descriptions.
200  */
201 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
202 {
203 	unsigned int max_usage = READ_ONCE(pipe->max_usage);
204 
205 	spd->nr_pages_max = max_usage;
206 	if (max_usage <= PIPE_DEF_BUFFERS)
207 		return 0;
208 
209 	spd->pages = kmalloc_objs(struct page *, max_usage);
210 	spd->partial = kmalloc_objs(struct partial_page, max_usage);
211 
212 	if (spd->pages && spd->partial)
213 		return 0;
214 
215 	kfree(spd->pages);
216 	kfree(spd->partial);
217 	return -ENOMEM;
218 }
219 
220 void splice_shrink_spd(struct splice_pipe_desc *spd)
221 {
222 	if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
223 		return;
224 
225 	kfree(spd->pages);
226 	kfree(spd->partial);
227 }
228 
229 /**
230  * copy_splice_read -  Copy data from a file and splice the copy into a pipe
231  * @in: The file to read from
232  * @ppos: Pointer to the file position to read from
233  * @pipe: The pipe to splice into
234  * @len: The amount to splice
235  * @flags: The SPLICE_F_* flags
236  *
237  * This function allocates a bunch of pages sufficient to hold the requested
238  * amount of data (but limited by the remaining pipe capacity), passes it to
239  * the file's ->read_iter() to read into and then splices the used pages into
240  * the pipe.
241  *
242  * Return: On success, the number of bytes read will be returned and *@ppos
243  * will be updated if appropriate; 0 will be returned if there is no more data
244  * to be read; -EAGAIN will be returned if the pipe had no space, and some
245  * other negative error code will be returned on error.  A short read may occur
246  * if the pipe has insufficient space, we reach the end of the data or we hit a
247  * hole.
248  */
249 ssize_t copy_splice_read(struct file *in, loff_t *ppos,
250 			 struct pipe_inode_info *pipe,
251 			 size_t len, unsigned int flags)
252 {
253 	struct iov_iter to;
254 	struct bio_vec *bv;
255 	struct kiocb kiocb;
256 	struct page **pages;
257 	ssize_t ret;
258 	size_t used, npages, chunk, remain, keep = 0;
259 	int i;
260 
261 	/* Work out how much data we can actually add into the pipe */
262 	used = pipe_buf_usage(pipe);
263 	npages = max_t(ssize_t, pipe->max_usage - used, 0);
264 	len = min_t(size_t, len, npages * PAGE_SIZE);
265 	npages = DIV_ROUND_UP(len, PAGE_SIZE);
266 
267 	bv = kzalloc(array_size(npages, sizeof(bv[0])) +
268 		     array_size(npages, sizeof(struct page *)), GFP_KERNEL);
269 	if (!bv)
270 		return -ENOMEM;
271 
272 	pages = (struct page **)(bv + npages);
273 	npages = alloc_pages_bulk(GFP_USER, npages, pages);
274 	if (!npages) {
275 		kfree(bv);
276 		return -ENOMEM;
277 	}
278 
279 	remain = len = min_t(size_t, len, npages * PAGE_SIZE);
280 
281 	for (i = 0; i < npages; i++) {
282 		chunk = min_t(size_t, PAGE_SIZE, remain);
283 		bv[i].bv_page = pages[i];
284 		bv[i].bv_offset = 0;
285 		bv[i].bv_len = chunk;
286 		remain -= chunk;
287 	}
288 
289 	/* Do the I/O */
290 	iov_iter_bvec(&to, ITER_DEST, bv, npages, len);
291 	init_sync_kiocb(&kiocb, in);
292 	kiocb.ki_pos = *ppos;
293 	ret = in->f_op->read_iter(&kiocb, &to);
294 
295 	if (ret > 0) {
296 		keep = DIV_ROUND_UP(ret, PAGE_SIZE);
297 		*ppos = kiocb.ki_pos;
298 	}
299 
300 	/*
301 	 * Callers of ->splice_read() expect -EAGAIN on "can't put anything in
302 	 * there", rather than -EFAULT.
303 	 */
304 	if (ret == -EFAULT)
305 		ret = -EAGAIN;
306 
307 	/* Free any pages that didn't get touched at all. */
308 	if (keep < npages)
309 		release_pages(pages + keep, npages - keep);
310 
311 	/* Push the remaining pages into the pipe. */
312 	remain = ret;
313 	for (i = 0; i < keep; i++) {
314 		struct pipe_buffer *buf = pipe_head_buf(pipe);
315 
316 		chunk = min_t(size_t, remain, PAGE_SIZE);
317 		*buf = (struct pipe_buffer) {
318 			.ops	= &default_pipe_buf_ops,
319 			.page	= bv[i].bv_page,
320 			.offset	= 0,
321 			.len	= chunk,
322 		};
323 		pipe->head++;
324 		remain -= chunk;
325 	}
326 
327 	kfree(bv);
328 	return ret;
329 }
330 EXPORT_SYMBOL(copy_splice_read);
331 
332 const struct pipe_buf_operations default_pipe_buf_ops = {
333 	.release	= generic_pipe_buf_release,
334 	.get		= generic_pipe_buf_get,
335 };
336 EXPORT_SYMBOL(default_pipe_buf_ops);
337 
338 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
339 {
340 	if (wq_has_sleeper(&pipe->wr_wait))
341 		wake_up_interruptible_poll(&pipe->wr_wait,
342 					   EPOLLOUT | EPOLLWRNORM);
343 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
344 }
345 
346 /**
347  * splice_from_pipe_feed - feed available data from a pipe to a file
348  * @pipe:	pipe to splice from
349  * @sd:		information to @actor
350  * @actor:	handler that splices the data
351  *
352  * Description:
353  *    This function loops over the pipe and calls @actor to do the
354  *    actual moving of a single struct pipe_buffer to the desired
355  *    destination.  It returns when there's no more buffers left in
356  *    the pipe or if the requested number of bytes (@sd->total_len)
357  *    have been copied.  It returns a positive number (one) if the
358  *    pipe needs to be filled with more data, zero if the required
359  *    number of bytes have been copied and -errno on error.
360  *
361  *    This, together with splice_from_pipe_{begin,end,next}, may be
362  *    used to implement the functionality of __splice_from_pipe() when
363  *    locking is required around copying the pipe buffers to the
364  *    destination.
365  */
366 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
367 			  splice_actor *actor)
368 {
369 	unsigned int head = pipe->head;
370 	unsigned int tail = pipe->tail;
371 	int ret;
372 
373 	while (!pipe_empty(head, tail)) {
374 		struct pipe_buffer *buf = pipe_buf(pipe, tail);
375 
376 		sd->len = buf->len;
377 		if (sd->len > sd->total_len)
378 			sd->len = sd->total_len;
379 
380 		ret = pipe_buf_confirm(pipe, buf);
381 		if (unlikely(ret)) {
382 			if (ret == -ENODATA)
383 				ret = 0;
384 			return ret;
385 		}
386 
387 		ret = actor(pipe, buf, sd);
388 		if (ret <= 0)
389 			return ret;
390 
391 		buf->offset += ret;
392 		buf->len -= ret;
393 
394 		sd->num_spliced += ret;
395 		sd->len -= ret;
396 		sd->pos += ret;
397 		sd->total_len -= ret;
398 
399 		if (!buf->len) {
400 			pipe_buf_release(pipe, buf);
401 			tail++;
402 			pipe->tail = tail;
403 			if (pipe->files)
404 				sd->need_wakeup = true;
405 		}
406 
407 		if (!sd->total_len)
408 			return 0;
409 	}
410 
411 	return 1;
412 }
413 
414 /* We know we have a pipe buffer, but maybe it's empty? */
415 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
416 {
417 	unsigned int tail = pipe->tail;
418 	struct pipe_buffer *buf = pipe_buf(pipe, tail);
419 
420 	if (unlikely(!buf->len)) {
421 		pipe_buf_release(pipe, buf);
422 		pipe->tail = tail+1;
423 		return true;
424 	}
425 
426 	return false;
427 }
428 
429 /**
430  * splice_from_pipe_next - wait for some data to splice from
431  * @pipe:	pipe to splice from
432  * @sd:		information about the splice operation
433  *
434  * Description:
435  *    This function will wait for some data and return a positive
436  *    value (one) if pipe buffers are available.  It will return zero
437  *    or -errno if no more data needs to be spliced.
438  */
439 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
440 {
441 	/*
442 	 * Check for signal early to make process killable when there are
443 	 * always buffers available
444 	 */
445 	if (signal_pending(current))
446 		return -ERESTARTSYS;
447 
448 repeat:
449 	while (pipe_is_empty(pipe)) {
450 		if (!pipe->writers)
451 			return 0;
452 
453 		if (sd->num_spliced)
454 			return 0;
455 
456 		if (sd->flags & SPLICE_F_NONBLOCK)
457 			return -EAGAIN;
458 
459 		if (signal_pending(current))
460 			return -ERESTARTSYS;
461 
462 		if (sd->need_wakeup) {
463 			wakeup_pipe_writers(pipe);
464 			sd->need_wakeup = false;
465 		}
466 
467 		pipe_wait_readable(pipe);
468 	}
469 
470 	if (eat_empty_buffer(pipe))
471 		goto repeat;
472 
473 	return 1;
474 }
475 
476 /**
477  * splice_from_pipe_begin - start splicing from pipe
478  * @sd:		information about the splice operation
479  *
480  * Description:
481  *    This function should be called before a loop containing
482  *    splice_from_pipe_next() and splice_from_pipe_feed() to
483  *    initialize the necessary fields of @sd.
484  */
485 static void splice_from_pipe_begin(struct splice_desc *sd)
486 {
487 	sd->num_spliced = 0;
488 	sd->need_wakeup = false;
489 }
490 
491 /**
492  * splice_from_pipe_end - finish splicing from pipe
493  * @pipe:	pipe to splice from
494  * @sd:		information about the splice operation
495  *
496  * Description:
497  *    This function will wake up pipe writers if necessary.  It should
498  *    be called after a loop containing splice_from_pipe_next() and
499  *    splice_from_pipe_feed().
500  */
501 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
502 {
503 	if (sd->need_wakeup)
504 		wakeup_pipe_writers(pipe);
505 }
506 
507 /**
508  * __splice_from_pipe - splice data from a pipe to given actor
509  * @pipe:	pipe to splice from
510  * @sd:		information to @actor
511  * @actor:	handler that splices the data
512  *
513  * Description:
514  *    This function does little more than loop over the pipe and call
515  *    @actor to do the actual moving of a single struct pipe_buffer to
516  *    the desired destination. See pipe_to_file, pipe_to_sendmsg, or
517  *    pipe_to_user.
518  *
519  */
520 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
521 			   splice_actor *actor)
522 {
523 	int ret;
524 
525 	splice_from_pipe_begin(sd);
526 	do {
527 		cond_resched();
528 		ret = splice_from_pipe_next(pipe, sd);
529 		if (ret > 0)
530 			ret = splice_from_pipe_feed(pipe, sd, actor);
531 	} while (ret > 0);
532 	splice_from_pipe_end(pipe, sd);
533 
534 	return sd->num_spliced ? sd->num_spliced : ret;
535 }
536 EXPORT_SYMBOL(__splice_from_pipe);
537 
538 /**
539  * splice_from_pipe - splice data from a pipe to a file
540  * @pipe:	pipe to splice from
541  * @out:	file to splice to
542  * @ppos:	position in @out
543  * @len:	how many bytes to splice
544  * @flags:	splice modifier flags
545  * @actor:	handler that splices the data
546  *
547  * Description:
548  *    See __splice_from_pipe. This function locks the pipe inode,
549  *    otherwise it's identical to __splice_from_pipe().
550  *
551  */
552 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
553 			 loff_t *ppos, size_t len, unsigned int flags,
554 			 splice_actor *actor)
555 {
556 	ssize_t ret;
557 	struct splice_desc sd = {
558 		.total_len = len,
559 		.flags = flags,
560 		.pos = *ppos,
561 		.u.file = out,
562 	};
563 
564 	pipe_lock(pipe);
565 	ret = __splice_from_pipe(pipe, &sd, actor);
566 	pipe_unlock(pipe);
567 
568 	return ret;
569 }
570 
571 /**
572  * iter_file_splice_write - splice data from a pipe to a file
573  * @pipe:	pipe info
574  * @out:	file to write to
575  * @ppos:	position in @out
576  * @len:	number of bytes to splice
577  * @flags:	splice modifier flags
578  *
579  * Description:
580  *    Will either move or copy pages (determined by @flags options) from
581  *    the given pipe inode to the given file.
582  *    This one is ->write_iter-based.
583  *
584  */
585 ssize_t
586 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
587 			  loff_t *ppos, size_t len, unsigned int flags)
588 {
589 	struct splice_desc sd = {
590 		.total_len = len,
591 		.flags = flags,
592 		.pos = *ppos,
593 		.u.file = out,
594 	};
595 	int nbufs = pipe->max_usage;
596 	struct bio_vec *array;
597 	ssize_t ret;
598 
599 	if (!out->f_op->write_iter)
600 		return -EINVAL;
601 
602 	array = kzalloc_objs(struct bio_vec, nbufs);
603 	if (unlikely(!array))
604 		return -ENOMEM;
605 
606 	pipe_lock(pipe);
607 
608 	splice_from_pipe_begin(&sd);
609 	while (sd.total_len) {
610 		struct kiocb kiocb;
611 		struct iov_iter from;
612 		unsigned int head, tail;
613 		size_t left;
614 		int n;
615 
616 		ret = splice_from_pipe_next(pipe, &sd);
617 		if (ret <= 0)
618 			break;
619 
620 		if (unlikely(nbufs < pipe->max_usage)) {
621 			kfree(array);
622 			nbufs = pipe->max_usage;
623 			array = kzalloc_objs(struct bio_vec, nbufs);
624 			if (!array) {
625 				ret = -ENOMEM;
626 				break;
627 			}
628 		}
629 
630 		head = pipe->head;
631 		tail = pipe->tail;
632 
633 		/* build the vector */
634 		left = sd.total_len;
635 		for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
636 			struct pipe_buffer *buf = pipe_buf(pipe, tail);
637 			size_t this_len = buf->len;
638 
639 			/* zero-length bvecs are not supported, skip them */
640 			if (!this_len)
641 				continue;
642 			this_len = min(this_len, left);
643 
644 			ret = pipe_buf_confirm(pipe, buf);
645 			if (unlikely(ret)) {
646 				if (ret == -ENODATA)
647 					ret = 0;
648 				goto done;
649 			}
650 
651 			bvec_set_page(&array[n], buf->page, this_len,
652 				      buf->offset);
653 			left -= this_len;
654 			n++;
655 		}
656 
657 		iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left);
658 		init_sync_kiocb(&kiocb, out);
659 		kiocb.ki_pos = sd.pos;
660 		ret = out->f_op->write_iter(&kiocb, &from);
661 		sd.pos = kiocb.ki_pos;
662 		if (ret <= 0)
663 			break;
664 		if (unlikely(ret > sd.total_len - left)) {
665 			WARN_ONCE(1,
666 				  "Splice Exceeded! ret=%zd tot=%zu left=%zu\n",
667 				  ret, sd.total_len, left);
668 			ret = sd.total_len - left;
669 		}
670 
671 		sd.num_spliced += ret;
672 		sd.total_len -= ret;
673 		*ppos = sd.pos;
674 
675 		/* dismiss the fully eaten buffers, adjust the partial one */
676 		tail = pipe->tail;
677 		while (ret) {
678 			struct pipe_buffer *buf = pipe_buf(pipe, tail);
679 			if (ret >= buf->len) {
680 				ret -= buf->len;
681 				buf->len = 0;
682 				pipe_buf_release(pipe, buf);
683 				tail++;
684 				pipe->tail = tail;
685 				if (pipe->files)
686 					sd.need_wakeup = true;
687 			} else {
688 				buf->offset += ret;
689 				buf->len -= ret;
690 				ret = 0;
691 			}
692 		}
693 	}
694 done:
695 	kfree(array);
696 	splice_from_pipe_end(pipe, &sd);
697 
698 	pipe_unlock(pipe);
699 
700 	if (sd.num_spliced)
701 		ret = sd.num_spliced;
702 
703 	return ret;
704 }
705 
706 EXPORT_SYMBOL(iter_file_splice_write);
707 
708 #ifdef CONFIG_NET
709 /**
710  * splice_to_socket - splice data from a pipe to a socket
711  * @pipe:	pipe to splice from
712  * @out:	socket to write to
713  * @ppos:	position in @out
714  * @len:	number of bytes to splice
715  * @flags:	splice modifier flags
716  *
717  * Description:
718  *    Will send @len bytes from the pipe to a network socket. No data copying
719  *    is involved.
720  *
721  */
722 ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out,
723 			 loff_t *ppos, size_t len, unsigned int flags)
724 {
725 	struct socket *sock = sock_from_file(out);
726 	struct bio_vec bvec[16];
727 	struct msghdr msg = {};
728 	ssize_t ret = 0;
729 	size_t spliced = 0;
730 	bool need_wakeup = false;
731 
732 	pipe_lock(pipe);
733 
734 	while (len > 0) {
735 		unsigned int head, tail, bc = 0;
736 		size_t remain = len;
737 
738 		/*
739 		 * Check for signal early to make process killable when there
740 		 * are always buffers available
741 		 */
742 		ret = -ERESTARTSYS;
743 		if (signal_pending(current))
744 			break;
745 
746 		while (pipe_is_empty(pipe)) {
747 			ret = 0;
748 			if (!pipe->writers)
749 				goto out;
750 
751 			if (spliced)
752 				goto out;
753 
754 			ret = -EAGAIN;
755 			if (flags & SPLICE_F_NONBLOCK)
756 				goto out;
757 
758 			ret = -ERESTARTSYS;
759 			if (signal_pending(current))
760 				goto out;
761 
762 			if (need_wakeup) {
763 				wakeup_pipe_writers(pipe);
764 				need_wakeup = false;
765 			}
766 
767 			pipe_wait_readable(pipe);
768 		}
769 
770 		head = pipe->head;
771 		tail = pipe->tail;
772 
773 		while (!pipe_empty(head, tail)) {
774 			struct pipe_buffer *buf = pipe_buf(pipe, tail);
775 			size_t seg;
776 
777 			if (!buf->len) {
778 				tail++;
779 				continue;
780 			}
781 
782 			seg = min_t(size_t, remain, buf->len);
783 
784 			ret = pipe_buf_confirm(pipe, buf);
785 			if (unlikely(ret)) {
786 				if (ret == -ENODATA)
787 					ret = 0;
788 				break;
789 			}
790 
791 			bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset);
792 			remain -= seg;
793 			tail++;
794 			if (remain == 0 || bc >= ARRAY_SIZE(bvec))
795 				break;
796 		}
797 
798 		if (!bc)
799 			break;
800 
801 		msg.msg_flags = MSG_SPLICE_PAGES;
802 		if (flags & SPLICE_F_MORE)
803 			msg.msg_flags |= MSG_MORE;
804 		if (remain && pipe_occupancy(pipe->head, tail) > 0)
805 			msg.msg_flags |= MSG_MORE;
806 		if (out->f_flags & O_NONBLOCK)
807 			msg.msg_flags |= MSG_DONTWAIT;
808 
809 		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bc,
810 			      len - remain);
811 		ret = sock_sendmsg(sock, &msg);
812 		if (ret <= 0)
813 			break;
814 
815 		spliced += ret;
816 		len -= ret;
817 		tail = pipe->tail;
818 		while (ret > 0) {
819 			struct pipe_buffer *buf = pipe_buf(pipe, tail);
820 			size_t seg = min_t(size_t, ret, buf->len);
821 
822 			buf->offset += seg;
823 			buf->len -= seg;
824 			ret -= seg;
825 
826 			if (!buf->len) {
827 				pipe_buf_release(pipe, buf);
828 				tail++;
829 			}
830 		}
831 
832 		if (tail != pipe->tail) {
833 			pipe->tail = tail;
834 			if (pipe->files)
835 				need_wakeup = true;
836 		}
837 	}
838 
839 out:
840 	pipe_unlock(pipe);
841 	if (need_wakeup)
842 		wakeup_pipe_writers(pipe);
843 	return spliced ?: ret;
844 }
845 #endif
846 
847 static int warn_unsupported(struct file *file, const char *op)
848 {
849 	pr_debug_ratelimited(
850 		"splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
851 		op, file, current->pid, current->comm);
852 	return -EINVAL;
853 }
854 
855 /*
856  * Attempt to initiate a splice from pipe to file.
857  */
858 static ssize_t do_splice_from(struct pipe_inode_info *pipe, struct file *out,
859 			      loff_t *ppos, size_t len, unsigned int flags)
860 {
861 	if (unlikely(!out->f_op->splice_write))
862 		return warn_unsupported(out, "write");
863 	return out->f_op->splice_write(pipe, out, ppos, len, flags);
864 }
865 
866 /*
867  * Indicate to the caller that there was a premature EOF when reading from the
868  * source and the caller didn't indicate they would be sending more data after
869  * this.
870  */
871 static void do_splice_eof(struct splice_desc *sd)
872 {
873 	if (sd->splice_eof)
874 		sd->splice_eof(sd);
875 }
876 
877 /*
878  * Callers already called rw_verify_area() on the entire range.
879  * No need to call it for sub ranges.
880  */
881 static ssize_t do_splice_read(struct file *in, loff_t *ppos,
882 			      struct pipe_inode_info *pipe, size_t len,
883 			      unsigned int flags)
884 {
885 	unsigned int p_space;
886 
887 	if (unlikely(!(in->f_mode & FMODE_READ)))
888 		return -EBADF;
889 	if (!len)
890 		return 0;
891 
892 	/* Don't try to read more the pipe has space for. */
893 	p_space = pipe->max_usage - pipe_buf_usage(pipe);
894 	len = min_t(size_t, len, p_space << PAGE_SHIFT);
895 
896 	if (unlikely(len > MAX_RW_COUNT))
897 		len = MAX_RW_COUNT;
898 
899 	if (unlikely(!in->f_op->splice_read))
900 		return warn_unsupported(in, "read");
901 	/*
902 	 * O_DIRECT and DAX don't deal with the pagecache, so we allocate a
903 	 * buffer, copy into it and splice that into the pipe.
904 	 */
905 	if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host))
906 		return copy_splice_read(in, ppos, pipe, len, flags);
907 	return in->f_op->splice_read(in, ppos, pipe, len, flags);
908 }
909 
910 /**
911  * vfs_splice_read - Read data from a file and splice it into a pipe
912  * @in:		File to splice from
913  * @ppos:	Input file offset
914  * @pipe:	Pipe to splice to
915  * @len:	Number of bytes to splice
916  * @flags:	Splice modifier flags (SPLICE_F_*)
917  *
918  * Splice the requested amount of data from the input file to the pipe.  This
919  * is synchronous as the caller must hold the pipe lock across the entire
920  * operation.
921  *
922  * If successful, it returns the amount of data spliced, 0 if it hit the EOF or
923  * a hole and a negative error code otherwise.
924  */
925 ssize_t vfs_splice_read(struct file *in, loff_t *ppos,
926 			struct pipe_inode_info *pipe, size_t len,
927 			unsigned int flags)
928 {
929 	ssize_t ret;
930 
931 	ret = rw_verify_area(READ, in, ppos, len);
932 	if (unlikely(ret < 0))
933 		return ret;
934 
935 	return do_splice_read(in, ppos, pipe, len, flags);
936 }
937 EXPORT_SYMBOL_GPL(vfs_splice_read);
938 
939 /*
940  * This is a special case helper to splice directly between two
941  * points, without requiring an explicit pipe. Internally an allocated
942  * pipe is cached in the process, and reused during the lifetime of
943  * that process.
944  */
945 static ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
946 				      splice_direct_actor *actor)
947 {
948 	struct pipe_inode_info *pipe;
949 	ssize_t ret, bytes;
950 	size_t len;
951 	int i, flags, more;
952 
953 	/*
954 	 * We require the input to be seekable, as we don't want to randomly
955 	 * drop data for eg socket -> socket splicing. Use the piped splicing
956 	 * for that!
957 	 */
958 	if (unlikely(!(in->f_mode & FMODE_LSEEK)))
959 		return -EINVAL;
960 
961 	/*
962 	 * neither in nor out is a pipe, setup an internal pipe attached to
963 	 * 'out' and transfer the wanted data from 'in' to 'out' through that
964 	 */
965 	pipe = current->splice_pipe;
966 	if (unlikely(!pipe)) {
967 		pipe = alloc_pipe_info();
968 		if (!pipe)
969 			return -ENOMEM;
970 
971 		/*
972 		 * We don't have an immediate reader, but we'll read the stuff
973 		 * out of the pipe right after the splice_to_pipe(). So set
974 		 * PIPE_READERS appropriately.
975 		 */
976 		pipe->readers = 1;
977 
978 		current->splice_pipe = pipe;
979 	}
980 
981 	/*
982 	 * Do the splice.
983 	 */
984 	bytes = 0;
985 	len = sd->total_len;
986 
987 	/* Don't block on output, we have to drain the direct pipe. */
988 	flags = sd->flags;
989 	sd->flags &= ~SPLICE_F_NONBLOCK;
990 
991 	/*
992 	 * We signal MORE until we've read sufficient data to fulfill the
993 	 * request and we keep signalling it if the caller set it.
994 	 */
995 	more = sd->flags & SPLICE_F_MORE;
996 	sd->flags |= SPLICE_F_MORE;
997 
998 	WARN_ON_ONCE(!pipe_is_empty(pipe));
999 
1000 	while (len) {
1001 		size_t read_len;
1002 		loff_t pos = sd->pos, prev_pos = pos;
1003 
1004 		ret = do_splice_read(in, &pos, pipe, len, flags);
1005 		if (unlikely(ret <= 0))
1006 			goto read_failure;
1007 
1008 		read_len = ret;
1009 		sd->total_len = read_len;
1010 
1011 		/*
1012 		 * If we now have sufficient data to fulfill the request then
1013 		 * we clear SPLICE_F_MORE if it was not set initially.
1014 		 */
1015 		if (read_len >= len && !more)
1016 			sd->flags &= ~SPLICE_F_MORE;
1017 
1018 		/*
1019 		 * NOTE: nonblocking mode only applies to the input. We
1020 		 * must not do the output in nonblocking mode as then we
1021 		 * could get stuck data in the internal pipe:
1022 		 */
1023 		ret = actor(pipe, sd);
1024 		if (unlikely(ret <= 0)) {
1025 			sd->pos = prev_pos;
1026 			goto out_release;
1027 		}
1028 
1029 		bytes += ret;
1030 		len -= ret;
1031 		sd->pos = pos;
1032 
1033 		if (ret < read_len) {
1034 			sd->pos = prev_pos + ret;
1035 			goto out_release;
1036 		}
1037 	}
1038 
1039 done:
1040 	pipe->tail = pipe->head = 0;
1041 	file_accessed(in);
1042 	return bytes;
1043 
1044 read_failure:
1045 	/*
1046 	 * If the user did *not* set SPLICE_F_MORE *and* we didn't hit that
1047 	 * "use all of len" case that cleared SPLICE_F_MORE, *and* we did a
1048 	 * "->splice_in()" that returned EOF (ie zero) *and* we have sent at
1049 	 * least 1 byte *then* we will also do the ->splice_eof() call.
1050 	 */
1051 	if (ret == 0 && !more && len > 0 && bytes)
1052 		do_splice_eof(sd);
1053 out_release:
1054 	/*
1055 	 * If we did an incomplete transfer we must release
1056 	 * the pipe buffers in question:
1057 	 */
1058 	for (i = 0; i < pipe->ring_size; i++) {
1059 		struct pipe_buffer *buf = &pipe->bufs[i];
1060 
1061 		if (buf->ops)
1062 			pipe_buf_release(pipe, buf);
1063 	}
1064 
1065 	if (!bytes)
1066 		bytes = ret;
1067 
1068 	goto done;
1069 }
1070 
1071 /**
1072  * vfs_splice_to_actor - call an actor on data read from a file
1073  * @in:		file to read from
1074  * @pos:	file offset
1075  * @count:	maximum number of bytes to read
1076  * @actor:	callback to process a pipe's worth of data
1077  * @private:	private data passed to @actor
1078  *
1079  * Read up to @count worth of data from @in at @pos, and call @actor
1080  * when the hidden pipe used to buffer the data is full. Ensures the
1081  * read is allowed using rw_verify_area() and emits fsnotify access
1082  * events. @in must be seekable (FMODE_LSEEK).
1083  *
1084  * Return: The number of bytes spliced, or a negative errno.
1085  */
1086 ssize_t vfs_splice_to_actor(struct file *in, loff_t pos, size_t count,
1087 			    splice_direct_actor *actor, void *private)
1088 {
1089 	struct splice_desc sd = {
1090 		.total_len	= count,
1091 		.pos		= pos,
1092 		.u.data		= private,
1093 	};
1094 	ssize_t ret;
1095 
1096 	ret = rw_verify_area(READ, in, &sd.pos, sd.total_len);
1097 	if (ret < 0)
1098 		return ret;
1099 
1100 	ret = splice_direct_to_actor(in, &sd, actor);
1101 	if (ret >= 0)
1102 		fsnotify_access(in);
1103 	return ret;
1104 }
1105 EXPORT_SYMBOL(vfs_splice_to_actor);
1106 
1107 static int direct_splice_actor(struct pipe_inode_info *pipe,
1108 			       struct splice_desc *sd)
1109 {
1110 	struct file *file = sd->u.file;
1111 	long ret;
1112 
1113 	file_start_write(file);
1114 	ret = do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
1115 	file_end_write(file);
1116 	return ret;
1117 }
1118 
1119 static int splice_file_range_actor(struct pipe_inode_info *pipe,
1120 					struct splice_desc *sd)
1121 {
1122 	struct file *file = sd->u.file;
1123 
1124 	return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);
1125 }
1126 
1127 static void direct_file_splice_eof(struct splice_desc *sd)
1128 {
1129 	struct file *file = sd->u.file;
1130 
1131 	if (file->f_op->splice_eof)
1132 		file->f_op->splice_eof(file);
1133 }
1134 
1135 static ssize_t do_splice_direct_actor(struct file *in, loff_t *ppos,
1136 				      struct file *out, loff_t *opos,
1137 				      size_t len, unsigned int flags,
1138 				      splice_direct_actor *actor)
1139 {
1140 	struct splice_desc sd = {
1141 		.len		= len,
1142 		.total_len	= len,
1143 		.flags		= flags,
1144 		.pos		= *ppos,
1145 		.u.file		= out,
1146 		.splice_eof	= direct_file_splice_eof,
1147 		.opos		= opos,
1148 	};
1149 	ssize_t ret;
1150 
1151 	if (unlikely(!(out->f_mode & FMODE_WRITE)))
1152 		return -EBADF;
1153 
1154 	if (unlikely(out->f_flags & O_APPEND))
1155 		return -EINVAL;
1156 
1157 	ret = splice_direct_to_actor(in, &sd, actor);
1158 	if (ret > 0)
1159 		*ppos = sd.pos;
1160 
1161 	return ret;
1162 }
1163 /**
1164  * do_splice_direct - splices data directly between two files
1165  * @in:		file to splice from
1166  * @ppos:	input file offset
1167  * @out:	file to splice to
1168  * @opos:	output file offset
1169  * @len:	number of bytes to splice
1170  * @flags:	splice modifier flags
1171  *
1172  * Description:
1173  *    For use by do_sendfile(). splice can easily emulate sendfile, but
1174  *    doing it in the application would incur an extra system call
1175  *    (splice in + splice out, as compared to just sendfile()). So this helper
1176  *    can splice directly through a process-private pipe.
1177  *
1178  * Callers already called rw_verify_area() on the entire range.
1179  */
1180 ssize_t do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1181 			 loff_t *opos, size_t len, unsigned int flags)
1182 {
1183 	return do_splice_direct_actor(in, ppos, out, opos, len, flags,
1184 				      direct_splice_actor);
1185 }
1186 EXPORT_SYMBOL(do_splice_direct);
1187 
1188 /**
1189  * splice_file_range - splices data between two files for copy_file_range()
1190  * @in:		file to splice from
1191  * @ppos:	input file offset
1192  * @out:	file to splice to
1193  * @opos:	output file offset
1194  * @len:	number of bytes to splice
1195  *
1196  * Description:
1197  *    For use by ->copy_file_range() methods.
1198  *    Like do_splice_direct(), but vfs_copy_file_range() already holds
1199  *    start_file_write() on @out file.
1200  *
1201  * Callers already called rw_verify_area() on the entire range.
1202  */
1203 ssize_t splice_file_range(struct file *in, loff_t *ppos, struct file *out,
1204 			  loff_t *opos, size_t len)
1205 {
1206 	lockdep_assert(file_write_started(out));
1207 
1208 	return do_splice_direct_actor(in, ppos, out, opos,
1209 				      min_t(size_t, len, MAX_RW_COUNT),
1210 				      0, splice_file_range_actor);
1211 }
1212 EXPORT_SYMBOL(splice_file_range);
1213 
1214 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
1215 {
1216 	for (;;) {
1217 		if (unlikely(!pipe->readers)) {
1218 			send_sig(SIGPIPE, current, 0);
1219 			return -EPIPE;
1220 		}
1221 		if (!pipe_is_full(pipe))
1222 			return 0;
1223 		if (flags & SPLICE_F_NONBLOCK)
1224 			return -EAGAIN;
1225 		if (signal_pending(current))
1226 			return -ERESTARTSYS;
1227 		pipe_wait_writable(pipe);
1228 	}
1229 }
1230 
1231 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1232 			       struct pipe_inode_info *opipe,
1233 			       size_t len, unsigned int flags);
1234 
1235 ssize_t splice_file_to_pipe(struct file *in,
1236 			    struct pipe_inode_info *opipe,
1237 			    loff_t *offset,
1238 			    size_t len, unsigned int flags)
1239 {
1240 	ssize_t ret;
1241 
1242 	pipe_lock(opipe);
1243 	ret = wait_for_space(opipe, flags);
1244 	if (!ret)
1245 		ret = do_splice_read(in, offset, opipe, len, flags);
1246 	pipe_unlock(opipe);
1247 	if (ret > 0)
1248 		wakeup_pipe_readers(opipe);
1249 	return ret;
1250 }
1251 
1252 /*
1253  * Determine where to splice to/from.
1254  */
1255 ssize_t do_splice(struct file *in, loff_t *off_in, struct file *out,
1256 		  loff_t *off_out, size_t len, unsigned int flags)
1257 {
1258 	struct pipe_inode_info *ipipe;
1259 	struct pipe_inode_info *opipe;
1260 	loff_t offset;
1261 	ssize_t ret;
1262 
1263 	if (unlikely(!(in->f_mode & FMODE_READ) ||
1264 		     !(out->f_mode & FMODE_WRITE)))
1265 		return -EBADF;
1266 
1267 	ipipe = get_pipe_info(in, true);
1268 	opipe = get_pipe_info(out, true);
1269 
1270 	if (ipipe && opipe) {
1271 		if (off_in || off_out)
1272 			return -ESPIPE;
1273 
1274 		/* Splicing to self would be fun, but... */
1275 		if (ipipe == opipe)
1276 			return -EINVAL;
1277 
1278 		if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1279 			flags |= SPLICE_F_NONBLOCK;
1280 
1281 		ret = splice_pipe_to_pipe(ipipe, opipe, len, flags);
1282 	} else if (ipipe) {
1283 		if (off_in)
1284 			return -ESPIPE;
1285 		if (off_out) {
1286 			if (!(out->f_mode & FMODE_PWRITE))
1287 				return -EINVAL;
1288 			offset = *off_out;
1289 		} else {
1290 			offset = out->f_pos;
1291 		}
1292 
1293 		if (unlikely(out->f_flags & O_APPEND))
1294 			return -EINVAL;
1295 
1296 		ret = rw_verify_area(WRITE, out, &offset, len);
1297 		if (unlikely(ret < 0))
1298 			return ret;
1299 
1300 		if (in->f_flags & O_NONBLOCK)
1301 			flags |= SPLICE_F_NONBLOCK;
1302 
1303 		file_start_write(out);
1304 		ret = do_splice_from(ipipe, out, &offset, len, flags);
1305 		file_end_write(out);
1306 
1307 		if (off_out)
1308 			*off_out = offset;
1309 		else if (ret >= 0)
1310 			out->f_pos = offset;
1311 	} else if (opipe) {
1312 		if (off_out)
1313 			return -ESPIPE;
1314 		if (off_in) {
1315 			if (!(in->f_mode & FMODE_PREAD))
1316 				return -EINVAL;
1317 			offset = *off_in;
1318 		} else {
1319 			offset = in->f_pos;
1320 		}
1321 
1322 		ret = rw_verify_area(READ, in, &offset, len);
1323 		if (unlikely(ret < 0))
1324 			return ret;
1325 
1326 		if (out->f_flags & O_NONBLOCK)
1327 			flags |= SPLICE_F_NONBLOCK;
1328 
1329 		ret = splice_file_to_pipe(in, opipe, &offset, len, flags);
1330 
1331 		if (off_in)
1332 			*off_in = offset;
1333 		else if (ret >= 0)
1334 			in->f_pos = offset;
1335 	} else {
1336 		ret = -EINVAL;
1337 	}
1338 
1339 	if (ret > 0) {
1340 		/*
1341 		 * Generate modify out before access in:
1342 		 * do_splice_from() may've already sent modify out,
1343 		 * and this ensures the events get merged.
1344 		 */
1345 		fsnotify_modify(out);
1346 		fsnotify_access(in);
1347 	}
1348 
1349 	return ret;
1350 }
1351 
1352 static ssize_t __do_splice(struct file *in, loff_t __user *off_in,
1353 			   struct file *out, loff_t __user *off_out,
1354 			   size_t len, unsigned int flags)
1355 {
1356 	struct pipe_inode_info *ipipe;
1357 	struct pipe_inode_info *opipe;
1358 	loff_t offset, *__off_in = NULL, *__off_out = NULL;
1359 	ssize_t ret;
1360 
1361 	ipipe = get_pipe_info(in, true);
1362 	opipe = get_pipe_info(out, true);
1363 
1364 	if (ipipe) {
1365 		if (off_in)
1366 			return -ESPIPE;
1367 		pipe_clear_nowait(in);
1368 	}
1369 	if (opipe) {
1370 		if (off_out)
1371 			return -ESPIPE;
1372 		pipe_clear_nowait(out);
1373 	}
1374 
1375 	if (off_out) {
1376 		if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1377 			return -EFAULT;
1378 		__off_out = &offset;
1379 	}
1380 	if (off_in) {
1381 		if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1382 			return -EFAULT;
1383 		__off_in = &offset;
1384 	}
1385 
1386 	ret = do_splice(in, __off_in, out, __off_out, len, flags);
1387 	if (ret < 0)
1388 		return ret;
1389 
1390 	if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
1391 		return -EFAULT;
1392 	if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
1393 		return -EFAULT;
1394 
1395 	return ret;
1396 }
1397 
1398 static ssize_t iter_to_pipe(struct iov_iter *from,
1399 			    struct pipe_inode_info *pipe)
1400 {
1401 	struct pipe_buffer buf = {
1402 		.ops = &user_page_pipe_buf_ops,
1403 	};
1404 	size_t total = 0;
1405 	ssize_t ret = 0;
1406 
1407 	while (iov_iter_count(from)) {
1408 		struct page *pages[16];
1409 		ssize_t left;
1410 		size_t start;
1411 		int i, n;
1412 
1413 		left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start);
1414 		if (left <= 0) {
1415 			ret = left;
1416 			break;
1417 		}
1418 
1419 		n = DIV_ROUND_UP(left + start, PAGE_SIZE);
1420 		for (i = 0; i < n; i++) {
1421 			int size = umin(left, PAGE_SIZE - start);
1422 
1423 			buf.page = pages[i];
1424 			buf.offset = start;
1425 			buf.len = size;
1426 			ret = add_to_pipe(pipe, &buf);
1427 			if (unlikely(ret < 0)) {
1428 				iov_iter_revert(from, left);
1429 				// this one got dropped by add_to_pipe()
1430 				while (++i < n)
1431 					put_page(pages[i]);
1432 				goto out;
1433 			}
1434 			total += ret;
1435 			left -= size;
1436 			start = 0;
1437 		}
1438 	}
1439 out:
1440 	return total ? total : ret;
1441 }
1442 
1443 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1444 			struct splice_desc *sd)
1445 {
1446 	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
1447 	return n == sd->len ? n : -EFAULT;
1448 }
1449 
1450 /*
1451  * For lack of a better implementation, implement vmsplice() to userspace
1452  * as a simple copy of the pipe's pages to the user iov.
1453  */
1454 static ssize_t vmsplice_to_user(struct file *file, struct iov_iter *iter,
1455 				unsigned int flags)
1456 {
1457 	struct pipe_inode_info *pipe = get_pipe_info(file, true);
1458 	struct splice_desc sd = {
1459 		.total_len = iov_iter_count(iter),
1460 		.flags = flags,
1461 		.u.data = iter
1462 	};
1463 	ssize_t ret = 0;
1464 
1465 	if (!pipe)
1466 		return -EBADF;
1467 
1468 	pipe_clear_nowait(file);
1469 
1470 	if (sd.total_len) {
1471 		pipe_lock(pipe);
1472 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1473 		pipe_unlock(pipe);
1474 	}
1475 
1476 	if (ret > 0)
1477 		fsnotify_access(file);
1478 
1479 	return ret;
1480 }
1481 
1482 /*
1483  * vmsplice splices a user address range into a pipe. It can be thought of
1484  * as splice-from-memory, where the regular splice is splice-from-file (or
1485  * to file). In both cases the output is a pipe, naturally.
1486  */
1487 static ssize_t vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
1488 				unsigned int flags)
1489 {
1490 	struct pipe_inode_info *pipe;
1491 	ssize_t ret = 0;
1492 
1493 	pipe = get_pipe_info(file, true);
1494 	if (!pipe)
1495 		return -EBADF;
1496 
1497 	pipe_clear_nowait(file);
1498 
1499 	pipe_lock(pipe);
1500 	ret = wait_for_space(pipe, flags);
1501 	if (!ret)
1502 		ret = iter_to_pipe(iter, pipe);
1503 	pipe_unlock(pipe);
1504 	if (ret > 0) {
1505 		wakeup_pipe_readers(pipe);
1506 		fsnotify_modify(file);
1507 	}
1508 	return ret;
1509 }
1510 
1511 /*
1512  * Note that vmsplice only really supports true splicing _from_ user memory
1513  * to a pipe, not the other way around. Splicing from user memory is a simple
1514  * operation that can be supported without any funky alignment restrictions
1515  * or nasty vm tricks. We simply map in the user memory and fill them into
1516  * a pipe. The reverse isn't quite as easy, though. There are two possible
1517  * solutions for that:
1518  *
1519  *	- memcpy() the data internally, at which point we might as well just
1520  *	  do a regular read() on the buffer anyway.
1521  *	- Lots of nasty vm tricks, that are neither fast nor flexible (it
1522  *	  has restriction limitations on both ends of the pipe).
1523  *
1524  * Currently we punt and implement it as a normal copy, see pipe_to_user().
1525  *
1526  */
1527 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
1528 		unsigned long, nr_segs, unsigned int, flags)
1529 {
1530 	struct iovec iovstack[UIO_FASTIOV];
1531 	struct iovec *iov = iovstack;
1532 	struct iov_iter iter;
1533 	ssize_t error;
1534 	int type;
1535 
1536 	if (unlikely(flags & ~SPLICE_F_ALL))
1537 		return -EINVAL;
1538 
1539 	CLASS(fd, f)(fd);
1540 	if (fd_empty(f))
1541 		return -EBADF;
1542 	if (fd_file(f)->f_mode & FMODE_WRITE)
1543 		type = ITER_SOURCE;
1544 	else if (fd_file(f)->f_mode & FMODE_READ)
1545 		type = ITER_DEST;
1546 	else
1547 		return -EBADF;
1548 
1549 	error = import_iovec(type, uiov, nr_segs,
1550 			     ARRAY_SIZE(iovstack), &iov, &iter);
1551 	if (error < 0)
1552 		return error;
1553 
1554 	if (!iov_iter_count(&iter))
1555 		error = 0;
1556 	else if (type == ITER_SOURCE)
1557 		error = vmsplice_to_pipe(fd_file(f), &iter, flags);
1558 	else
1559 		error = vmsplice_to_user(fd_file(f), &iter, flags);
1560 
1561 	kfree(iov);
1562 	return error;
1563 }
1564 
1565 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1566 		int, fd_out, loff_t __user *, off_out,
1567 		size_t, len, unsigned int, flags)
1568 {
1569 	if (unlikely(!len))
1570 		return 0;
1571 
1572 	if (unlikely(flags & ~SPLICE_F_ALL))
1573 		return -EINVAL;
1574 
1575 	CLASS(fd, in)(fd_in);
1576 	if (fd_empty(in))
1577 		return -EBADF;
1578 
1579 	CLASS(fd, out)(fd_out);
1580 	if (fd_empty(out))
1581 		return -EBADF;
1582 
1583 	return __do_splice(fd_file(in), off_in, fd_file(out), off_out,
1584 					    len, flags);
1585 }
1586 
1587 /*
1588  * Make sure there's data to read. Wait for input if we can, otherwise
1589  * return an appropriate error.
1590  */
1591 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1592 {
1593 	int ret;
1594 
1595 	/*
1596 	 * Check the pipe occupancy without the inode lock first. This function
1597 	 * is speculative anyways, so missing one is ok.
1598 	 */
1599 	if (!pipe_is_empty(pipe))
1600 		return 0;
1601 
1602 	ret = 0;
1603 	pipe_lock(pipe);
1604 
1605 	while (pipe_is_empty(pipe)) {
1606 		if (signal_pending(current)) {
1607 			ret = -ERESTARTSYS;
1608 			break;
1609 		}
1610 		if (!pipe->writers)
1611 			break;
1612 		if (flags & SPLICE_F_NONBLOCK) {
1613 			ret = -EAGAIN;
1614 			break;
1615 		}
1616 		pipe_wait_readable(pipe);
1617 	}
1618 
1619 	pipe_unlock(pipe);
1620 	return ret;
1621 }
1622 
1623 /*
1624  * Make sure there's writeable room. Wait for room if we can, otherwise
1625  * return an appropriate error.
1626  */
1627 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1628 {
1629 	int ret;
1630 
1631 	/*
1632 	 * Check pipe occupancy without the inode lock first. This function
1633 	 * is speculative anyways, so missing one is ok.
1634 	 */
1635 	if (!pipe_is_full(pipe))
1636 		return 0;
1637 
1638 	ret = 0;
1639 	pipe_lock(pipe);
1640 
1641 	while (pipe_is_full(pipe)) {
1642 		if (!pipe->readers) {
1643 			send_sig(SIGPIPE, current, 0);
1644 			ret = -EPIPE;
1645 			break;
1646 		}
1647 		if (flags & SPLICE_F_NONBLOCK) {
1648 			ret = -EAGAIN;
1649 			break;
1650 		}
1651 		if (signal_pending(current)) {
1652 			ret = -ERESTARTSYS;
1653 			break;
1654 		}
1655 		pipe_wait_writable(pipe);
1656 	}
1657 
1658 	pipe_unlock(pipe);
1659 	return ret;
1660 }
1661 
1662 /*
1663  * Splice contents of ipipe to opipe.
1664  */
1665 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1666 			       struct pipe_inode_info *opipe,
1667 			       size_t len, unsigned int flags)
1668 {
1669 	struct pipe_buffer *ibuf, *obuf;
1670 	unsigned int i_head, o_head;
1671 	unsigned int i_tail, o_tail;
1672 	int ret = 0;
1673 	bool input_wakeup = false;
1674 
1675 
1676 retry:
1677 	ret = ipipe_prep(ipipe, flags);
1678 	if (ret)
1679 		return ret;
1680 
1681 	ret = opipe_prep(opipe, flags);
1682 	if (ret)
1683 		return ret;
1684 
1685 	/*
1686 	 * Potential ABBA deadlock, work around it by ordering lock
1687 	 * grabbing by pipe info address. Otherwise two different processes
1688 	 * could deadlock (one doing tee from A -> B, the other from B -> A).
1689 	 */
1690 	pipe_double_lock(ipipe, opipe);
1691 
1692 	i_tail = ipipe->tail;
1693 	o_head = opipe->head;
1694 
1695 	do {
1696 		size_t o_len;
1697 
1698 		if (!opipe->readers) {
1699 			send_sig(SIGPIPE, current, 0);
1700 			if (!ret)
1701 				ret = -EPIPE;
1702 			break;
1703 		}
1704 
1705 		i_head = ipipe->head;
1706 		o_tail = opipe->tail;
1707 
1708 		if (pipe_empty(i_head, i_tail) && !ipipe->writers)
1709 			break;
1710 
1711 		/*
1712 		 * Cannot make any progress, because either the input
1713 		 * pipe is empty or the output pipe is full.
1714 		 */
1715 		if (pipe_empty(i_head, i_tail) ||
1716 		    pipe_full(o_head, o_tail, opipe->max_usage)) {
1717 			/* Already processed some buffers, break */
1718 			if (ret)
1719 				break;
1720 
1721 			if (flags & SPLICE_F_NONBLOCK) {
1722 				ret = -EAGAIN;
1723 				break;
1724 			}
1725 
1726 			/*
1727 			 * We raced with another reader/writer and haven't
1728 			 * managed to process any buffers.  A zero return
1729 			 * value means EOF, so retry instead.
1730 			 */
1731 			pipe_unlock(ipipe);
1732 			pipe_unlock(opipe);
1733 			goto retry;
1734 		}
1735 
1736 		ibuf = pipe_buf(ipipe, i_tail);
1737 		obuf = pipe_buf(opipe, o_head);
1738 
1739 		if (len >= ibuf->len) {
1740 			/*
1741 			 * Simply move the whole buffer from ipipe to opipe
1742 			 */
1743 			*obuf = *ibuf;
1744 			ibuf->ops = NULL;
1745 			i_tail++;
1746 			ipipe->tail = i_tail;
1747 			input_wakeup = true;
1748 			o_len = obuf->len;
1749 			o_head++;
1750 			opipe->head = o_head;
1751 		} else {
1752 			/*
1753 			 * Get a reference to this pipe buffer,
1754 			 * so we can copy the contents over.
1755 			 */
1756 			if (!pipe_buf_get(ipipe, ibuf)) {
1757 				if (ret == 0)
1758 					ret = -EFAULT;
1759 				break;
1760 			}
1761 			*obuf = *ibuf;
1762 
1763 			/* The page is shared with ipipe: don't append to it */
1764 			obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1765 
1766 			obuf->len = len;
1767 			ibuf->offset += len;
1768 			ibuf->len -= len;
1769 			o_len = len;
1770 			o_head++;
1771 			opipe->head = o_head;
1772 		}
1773 		ret += o_len;
1774 		len -= o_len;
1775 	} while (len);
1776 
1777 	pipe_unlock(ipipe);
1778 	pipe_unlock(opipe);
1779 
1780 	/*
1781 	 * If we put data in the output pipe, wakeup any potential readers.
1782 	 */
1783 	if (ret > 0)
1784 		wakeup_pipe_readers(opipe);
1785 
1786 	if (input_wakeup)
1787 		wakeup_pipe_writers(ipipe);
1788 
1789 	return ret;
1790 }
1791 
1792 /*
1793  * Link contents of ipipe to opipe.
1794  */
1795 static ssize_t link_pipe(struct pipe_inode_info *ipipe,
1796 			 struct pipe_inode_info *opipe,
1797 			 size_t len, unsigned int flags)
1798 {
1799 	struct pipe_buffer *ibuf, *obuf;
1800 	unsigned int i_head, o_head;
1801 	unsigned int i_tail, o_tail;
1802 	ssize_t ret = 0;
1803 
1804 retry:
1805 	ret = ipipe_prep(ipipe, flags);
1806 	if (ret)
1807 		return ret;
1808 
1809 	ret = opipe_prep(opipe, flags);
1810 	if (ret)
1811 		return ret;
1812 
1813 	/*
1814 	 * Potential ABBA deadlock, work around it by ordering lock
1815 	 * grabbing by pipe info address. Otherwise two different processes
1816 	 * could deadlock (one doing tee from A -> B, the other from B -> A).
1817 	 */
1818 	pipe_double_lock(ipipe, opipe);
1819 
1820 	i_tail = ipipe->tail;
1821 	o_head = opipe->head;
1822 
1823 	do {
1824 		if (!opipe->readers) {
1825 			send_sig(SIGPIPE, current, 0);
1826 			if (!ret)
1827 				ret = -EPIPE;
1828 			break;
1829 		}
1830 
1831 		i_head = ipipe->head;
1832 		o_tail = opipe->tail;
1833 
1834 		if (pipe_empty(i_head, i_tail) && !ipipe->writers)
1835 			break;
1836 
1837 		/*
1838 		 * If we have iterated all input buffers or run out of
1839 		 * output room, break.
1840 		 */
1841 		if (pipe_empty(i_head, i_tail) ||
1842 		    pipe_full(o_head, o_tail, opipe->max_usage)) {
1843 			if (ret)
1844 				break;
1845 
1846 			if (flags & SPLICE_F_NONBLOCK) {
1847 				ret = -EAGAIN;
1848 				break;
1849 			}
1850 
1851 			/*
1852 			 * We raced with another reader/writer and haven't
1853 			 * linked any buffers. A zero return value means EOF,
1854 			 * so retry instead.
1855 			 */
1856 			pipe_unlock(ipipe);
1857 			pipe_unlock(opipe);
1858 			goto retry;
1859 		}
1860 
1861 		ibuf = pipe_buf(ipipe, i_tail);
1862 		obuf = pipe_buf(opipe, o_head);
1863 
1864 		/*
1865 		 * Get a reference to this pipe buffer,
1866 		 * so we can copy the contents over.
1867 		 */
1868 		if (!pipe_buf_get(ipipe, ibuf)) {
1869 			if (ret == 0)
1870 				ret = -EFAULT;
1871 			break;
1872 		}
1873 
1874 		*obuf = *ibuf;
1875 
1876 		/* The page is shared with ipipe: don't append to it */
1877 		obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1878 
1879 		if (obuf->len > len)
1880 			obuf->len = len;
1881 		ret += obuf->len;
1882 		len -= obuf->len;
1883 
1884 		o_head++;
1885 		opipe->head = o_head;
1886 		i_tail++;
1887 	} while (len);
1888 
1889 	pipe_unlock(ipipe);
1890 	pipe_unlock(opipe);
1891 
1892 	/*
1893 	 * If we put data in the output pipe, wakeup any potential readers.
1894 	 */
1895 	if (ret > 0)
1896 		wakeup_pipe_readers(opipe);
1897 
1898 	return ret;
1899 }
1900 
1901 /*
1902  * This is a tee(1) implementation that works on pipes. It doesn't copy
1903  * any data, it simply references the 'in' pages on the 'out' pipe.
1904  * The 'flags' used are the SPLICE_F_* variants, currently the only
1905  * applicable one is SPLICE_F_NONBLOCK.
1906  */
1907 ssize_t do_tee(struct file *in, struct file *out, size_t len,
1908 	       unsigned int flags)
1909 {
1910 	struct pipe_inode_info *ipipe = get_pipe_info(in, true);
1911 	struct pipe_inode_info *opipe = get_pipe_info(out, true);
1912 	ssize_t ret = -EINVAL;
1913 
1914 	if (unlikely(!(in->f_mode & FMODE_READ) ||
1915 		     !(out->f_mode & FMODE_WRITE)))
1916 		return -EBADF;
1917 
1918 	/*
1919 	 * Duplicate the contents of ipipe to opipe without actually
1920 	 * copying the data.
1921 	 */
1922 	if (ipipe && opipe && ipipe != opipe) {
1923 		if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1924 			flags |= SPLICE_F_NONBLOCK;
1925 
1926 		ret = link_pipe(ipipe, opipe, len, flags);
1927 	}
1928 
1929 	if (ret > 0) {
1930 		fsnotify_access(in);
1931 		fsnotify_modify(out);
1932 	}
1933 
1934 	return ret;
1935 }
1936 
1937 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1938 {
1939 	if (unlikely(flags & ~SPLICE_F_ALL))
1940 		return -EINVAL;
1941 
1942 	if (unlikely(!len))
1943 		return 0;
1944 
1945 	CLASS(fd, in)(fdin);
1946 	if (fd_empty(in))
1947 		return -EBADF;
1948 
1949 	CLASS(fd, out)(fdout);
1950 	if (fd_empty(out))
1951 		return -EBADF;
1952 
1953 	return do_tee(fd_file(in), fd_file(out), len, flags);
1954 }
1955