64 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_try_steal()  argument
108 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_release()  argument
119 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,  in page_cache_pipe_buf_confirm()  argument
162 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,  in user_page_pipe_buf_try_steal()  argument
169 	return generic_pipe_buf_try_steal(pipe, buf);  in user_page_pipe_buf_try_steal()
178 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)  in wakeup_pipe_readers()  argument
181 	if (waitqueue_active(&pipe->rd_wait))  in wakeup_pipe_readers()
182 		wake_up_interruptible(&pipe->rd_wait);  in wakeup_pipe_readers()
183 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);  in wakeup_pipe_readers()
197 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,  in splice_to_pipe()  argument
201 	unsigned int tail = pipe->tail;  in splice_to_pipe()
202 	unsigned int head = pipe->head;  in splice_to_pipe()
209 	if (unlikely(!pipe->readers)) {  in splice_to_pipe()
215 	while (!pipe_full(head, tail, pipe->max_usage)) {  in splice_to_pipe()
216 		struct pipe_buffer *buf = pipe_buf(pipe, head);  in splice_to_pipe()
226 		pipe->head = head;  in splice_to_pipe()
245 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)  in add_to_pipe()  argument
247 	unsigned int head = pipe->head;  in add_to_pipe()
248 	unsigned int tail = pipe->tail;  in add_to_pipe()
251 	if (unlikely(!pipe->readers)) {  in add_to_pipe()
254 	} else if (pipe_full(head, tail, pipe->max_usage)) {  in add_to_pipe()
257 		*pipe_buf(pipe, head) = *buf;  in add_to_pipe()
258 		pipe->head = head + 1;  in add_to_pipe()
261 	pipe_buf_release(pipe, buf);  in add_to_pipe()
270 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)  in splice_grow_spd()  argument
272 	unsigned int max_usage = READ_ONCE(pipe->max_usage);  in splice_grow_spd()
320 			 struct pipe_inode_info *pipe,  in copy_splice_read()  argument
332 	used = pipe_buf_usage(pipe);  in copy_splice_read()
333 	npages = max_t(ssize_t, pipe->max_usage - used, 0);  in copy_splice_read()
384 		struct pipe_buffer *buf = pipe_head_buf(pipe);  in copy_splice_read()
393 		pipe->head++;  in copy_splice_read()
415 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)  in wakeup_pipe_writers()  argument
418 	if (waitqueue_active(&pipe->wr_wait))  in wakeup_pipe_writers()
419 		wake_up_interruptible(&pipe->wr_wait);  in wakeup_pipe_writers()
420 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);  in wakeup_pipe_writers()
443 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
446 	unsigned int head = pipe->head;  in splice_from_pipe_feed()
447 	unsigned int tail = pipe->tail;  in splice_from_pipe_feed()
451 		struct pipe_buffer *buf = pipe_buf(pipe, tail);  in splice_from_pipe_feed()
457 		ret = pipe_buf_confirm(pipe, buf);  in splice_from_pipe_feed()
464 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
477 			pipe_buf_release(pipe, buf);  in splice_from_pipe_feed()
479 			pipe->tail = tail;  in splice_from_pipe_feed()
480 			if (pipe->files)  in splice_from_pipe_feed()
492 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)  in eat_empty_buffer()  argument
494 	unsigned int tail = pipe->tail;  in eat_empty_buffer()
495 	struct pipe_buffer *buf = pipe_buf(pipe, tail);  in eat_empty_buffer()
498 		pipe_buf_release(pipe, buf);  in eat_empty_buffer()
499 		pipe->tail = tail+1;  in eat_empty_buffer()
516 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
526 	while (pipe_is_empty(pipe)) {  in splice_from_pipe_next()
527 		if (!pipe->writers)  in splice_from_pipe_next()
540 			wakeup_pipe_writers(pipe);  in splice_from_pipe_next()
544 		pipe_wait_readable(pipe);  in splice_from_pipe_next()
547 	if (eat_empty_buffer(pipe))  in splice_from_pipe_next()
578 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
581 		wakeup_pipe_writers(pipe);  in splice_from_pipe_end()
597 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
605 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
607 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
609 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
629 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,  in splice_from_pipe()  argument
641 	pipe_lock(pipe);  in splice_from_pipe()
642 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
643 	pipe_unlock(pipe);  in splice_from_pipe()
663 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,  in iter_file_splice_write()  argument
672 	int nbufs = pipe->max_usage;  in iter_file_splice_write()
683 	pipe_lock(pipe);  in iter_file_splice_write()
693 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
697 		if (unlikely(nbufs < pipe->max_usage)) {  in iter_file_splice_write()
699 			nbufs = pipe->max_usage;  in iter_file_splice_write()
708 		head = pipe->head;  in iter_file_splice_write()
709 		tail = pipe->tail;  in iter_file_splice_write()
714 			struct pipe_buffer *buf = pipe_buf(pipe, tail);  in iter_file_splice_write()
722 			ret = pipe_buf_confirm(pipe, buf);  in iter_file_splice_write()
751 		tail = pipe->tail;  in iter_file_splice_write()
753 			struct pipe_buffer *buf = pipe_buf(pipe, tail);  in iter_file_splice_write()
757 				pipe_buf_release(pipe, buf);  in iter_file_splice_write()
759 				pipe->tail = tail;  in iter_file_splice_write()
760 				if (pipe->files)  in iter_file_splice_write()
771 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
773 	pipe_unlock(pipe);  in iter_file_splice_write()
797 ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out,  in splice_to_socket()  argument
807 	pipe_lock(pipe);  in splice_to_socket()
821 		while (pipe_is_empty(pipe)) {  in splice_to_socket()
823 			if (!pipe->writers)  in splice_to_socket()
838 				wakeup_pipe_writers(pipe);  in splice_to_socket()
842 			pipe_wait_readable(pipe);  in splice_to_socket()
845 		head = pipe->head;  in splice_to_socket()
846 		tail = pipe->tail;  in splice_to_socket()
849 			struct pipe_buffer *buf = pipe_buf(pipe, tail);  in splice_to_socket()
859 			ret = pipe_buf_confirm(pipe, buf);  in splice_to_socket()
879 		if (remain && pipe_occupancy(pipe->head, tail) > 0)  in splice_to_socket()
892 		tail = pipe->tail;  in splice_to_socket()
894 			struct pipe_buffer *buf = pipe_buf(pipe, tail);  in splice_to_socket()
902 				pipe_buf_release(pipe, buf);  in splice_to_socket()
907 		if (tail != pipe->tail) {  in splice_to_socket()
908 			pipe->tail = tail;  in splice_to_socket()
909 			if (pipe->files)  in splice_to_socket()
915 	pipe_unlock(pipe);  in splice_to_socket()
917 		wakeup_pipe_writers(pipe);  in splice_to_socket()
933 static ssize_t do_splice_from(struct pipe_inode_info *pipe, struct file *out,  in do_splice_from()  argument
938 	return out->f_op->splice_write(pipe, out, ppos, len, flags);  in do_splice_from()
957 			      struct pipe_inode_info *pipe, size_t len,  in do_splice_read()  argument
968 	p_space = pipe->max_usage - pipe_buf_usage(pipe);  in do_splice_read()
981 		return copy_splice_read(in, ppos, pipe, len, flags);  in do_splice_read()
982 	return in->f_op->splice_read(in, ppos, pipe, len, flags);  in do_splice_read()
1001 			struct pipe_inode_info *pipe, size_t len,  in vfs_splice_read()  argument
1010 	return do_splice_read(in, ppos, pipe, len, flags);  in vfs_splice_read()
1030 	struct pipe_inode_info *pipe;  in splice_direct_to_actor()  local
1047 	pipe = current->splice_pipe;  in splice_direct_to_actor()
1048 	if (unlikely(!pipe)) {  in splice_direct_to_actor()
1049 		pipe = alloc_pipe_info();  in splice_direct_to_actor()
1050 		if (!pipe)  in splice_direct_to_actor()
1058 		pipe->readers = 1;  in splice_direct_to_actor()
1060 		current->splice_pipe = pipe;  in splice_direct_to_actor()
1080 	WARN_ON_ONCE(!pipe_is_empty(pipe));  in splice_direct_to_actor()
1086 		ret = do_splice_read(in, &pos, pipe, len, flags);  in splice_direct_to_actor()
1105 		ret = actor(pipe, sd);  in splice_direct_to_actor()
1122 	pipe->tail = pipe->head = 0;  in splice_direct_to_actor()
1140 	for (i = 0; i < pipe->ring_size; i++) {  in splice_direct_to_actor()
1141 		struct pipe_buffer *buf = &pipe->bufs[i];  in splice_direct_to_actor()
1144 			pipe_buf_release(pipe, buf);  in splice_direct_to_actor()
1154 static int direct_splice_actor(struct pipe_inode_info *pipe,  in direct_splice_actor()  argument
1161 	ret = do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);  in direct_splice_actor()
1166 static int splice_file_range_actor(struct pipe_inode_info *pipe,  in splice_file_range_actor()  argument
1171 	return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags);  in splice_file_range_actor()
1261 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)  in wait_for_space()  argument
1264 		if (unlikely(!pipe->readers)) {  in wait_for_space()
1268 		if (!pipe_is_full(pipe))  in wait_for_space()
1274 		pipe_wait_writable(pipe);  in wait_for_space()
1446 			    struct pipe_inode_info *pipe,  in iter_to_pipe()  argument
1475 			ret = add_to_pipe(pipe, &buf);  in iter_to_pipe()
1492 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,  in pipe_to_user()  argument
1506 	struct pipe_inode_info *pipe = get_pipe_info(file, true);  in vmsplice_to_user()  local
1514 	if (!pipe)  in vmsplice_to_user()
1520 		pipe_lock(pipe);  in vmsplice_to_user()
1521 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()
1522 		pipe_unlock(pipe);  in vmsplice_to_user()
1539 	struct pipe_inode_info *pipe;  in vmsplice_to_pipe()  local
1546 	pipe = get_pipe_info(file, true);  in vmsplice_to_pipe()
1547 	if (!pipe)  in vmsplice_to_pipe()
1552 	pipe_lock(pipe);  in vmsplice_to_pipe()
1553 	ret = wait_for_space(pipe, flags);  in vmsplice_to_pipe()
1555 		ret = iter_to_pipe(iter, pipe, buf_flag);  in vmsplice_to_pipe()
1556 	pipe_unlock(pipe);  in vmsplice_to_pipe()
1558 		wakeup_pipe_readers(pipe);  in vmsplice_to_pipe()
1644 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in ipipe_prep()  argument
1652 	if (!pipe_is_empty(pipe))  in ipipe_prep()
1656 	pipe_lock(pipe);  in ipipe_prep()
1658 	while (pipe_is_empty(pipe)) {  in ipipe_prep()
1663 		if (!pipe->writers)  in ipipe_prep()
1669 		pipe_wait_readable(pipe);  in ipipe_prep()
1672 	pipe_unlock(pipe);  in ipipe_prep()
1680 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)  in opipe_prep()  argument
1688 	if (!pipe_is_full(pipe))  in opipe_prep()
1692 	pipe_lock(pipe);  in opipe_prep()
1694 	while (pipe_is_full(pipe)) {  in opipe_prep()
1695 		if (!pipe->readers) {  in opipe_prep()
1708 		pipe_wait_writable(pipe);  in opipe_prep()
1711 	pipe_unlock(pipe);  in opipe_prep()