xref: /linux/fs/pipe.c (revision 0f76da93e5c7d41ccf1376e715278c1d474ed181)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/fs/pipe.c
4  *
5  *  Copyright (C) 1991, 1992, 1999  Linus Torvalds
6  */
7 
8 #include <linux/mm.h>
9 #include <linux/file.h>
10 #include <linux/poll.h>
11 #include <linux/slab.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/fs.h>
15 #include <linux/log2.h>
16 #include <linux/mount.h>
17 #include <linux/pseudo_fs.h>
18 #include <linux/magic.h>
19 #include <linux/pipe_fs_i.h>
20 #include <linux/uio.h>
21 #include <linux/highmem.h>
22 #include <linux/pagemap.h>
23 #include <linux/audit.h>
24 #include <linux/syscalls.h>
25 #include <linux/fcntl.h>
26 #include <linux/memcontrol.h>
27 #include <linux/watch_queue.h>
28 #include <linux/sysctl.h>
29 #include <linux/sort.h>
30 
31 #include <linux/uaccess.h>
32 #include <asm/ioctls.h>
33 
34 #include "internal.h"
35 
36 /*
37  * New pipe buffers will be restricted to this size while the user is exceeding
38  * their pipe buffer quota. The general pipe use case needs at least two
39  * buffers: one for data yet to be read, and one for new data. If this is less
40  * than two, then a write to a non-empty pipe may block even if the pipe is not
41  * full. This can occur with GNU make jobserver or similar uses of pipes as
42  * semaphores: multiple processes may be waiting to write tokens back to the
43  * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
44  *
45  * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
46  * own risk, namely: pipe writes to non-full pipes may block until the pipe is
47  * emptied.
48  */
49 #define PIPE_MIN_DEF_BUFFERS 2
50 
51 /*
52  * The max size that a non-root user is allowed to grow the pipe. Can
53  * be set by root in /proc/sys/fs/pipe-max-size
54  */
55 static unsigned int pipe_max_size = 1048576;
56 
57 /* Maximum allocatable pages per user. Hard limit is unset by default, soft
58  * matches default values.
59  */
60 static unsigned long pipe_user_pages_hard;
61 static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
62 
63 /*
64  * We use head and tail indices that aren't masked off, except at the point of
65  * dereference, but rather they're allowed to wrap naturally.  This means there
66  * isn't a dead spot in the buffer, but the ring has to be a power of two and
67  * <= 2^31.
68  * -- David Howells 2019-09-23.
69  *
70  * Reads with count = 0 should always return 0.
71  * -- Julian Bradfield 1999-06-07.
72  *
73  * FIFOs and Pipes now generate SIGIO for both readers and writers.
74  * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
75  *
76  * pipe_read & write cleanup
77  * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
78  */
79 
80 #ifdef CONFIG_PROVE_LOCKING
81 static int pipe_lock_cmp_fn(const struct lockdep_map *a,
82 			    const struct lockdep_map *b)
83 {
84 	return cmp_int((unsigned long) a, (unsigned long) b);
85 }
86 #endif
87 
88 void pipe_lock(struct pipe_inode_info *pipe)
89 {
90 	if (pipe->files)
91 		mutex_lock(&pipe->mutex);
92 }
93 EXPORT_SYMBOL(pipe_lock);
94 
95 void pipe_unlock(struct pipe_inode_info *pipe)
96 {
97 	if (pipe->files)
98 		mutex_unlock(&pipe->mutex);
99 }
100 EXPORT_SYMBOL(pipe_unlock);
101 
102 void pipe_double_lock(struct pipe_inode_info *pipe1,
103 		      struct pipe_inode_info *pipe2)
104 {
105 	BUG_ON(pipe1 == pipe2);
106 
107 	if (pipe1 > pipe2)
108 		swap(pipe1, pipe2);
109 
110 	pipe_lock(pipe1);
111 	pipe_lock(pipe2);
112 }
113 
114 static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc)
115 {
116 	if (!prealloc->count)
117 		return NULL;
118 
119 	prealloc->count--;
120 
121 	return prealloc->pages[prealloc->count];
122 }
123 
124 /* Push a page to the prealloc pool. Returns true if added, false if full. */
125 static bool anon_pipe_prealloc_push(struct anon_pipe_prealloc *prealloc,
126 				    struct page *page)
127 {
128 	if (prealloc->count >= PIPE_PREALLOC_MAX)
129 		return false;
130 	prealloc->pages[prealloc->count++] = page;
131 	return true;
132 }
133 
134 /*
135  * Top up the pipe's own pool, then take pipe->mutex and return with it held.
136  * The shortfall is allocated outside the lock; the push and the caller's write
137  * then run under a single lock acquisition, avoiding a separate prefill
138  * lock/unlock cycle. anon_pipe_get_page() drains the pool instead of allocating
139  * under the lock.
140  */
141 static void anon_pipe_prefill_and_lock(struct pipe_inode_info *pipe, size_t total_len)
142 {
143 	struct page *pages[PIPE_PREALLOC_MAX];
144 	unsigned int want, have, need, n = 0;
145 
146 	want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE),
147 		     PIPE_PREALLOC_MAX);
148 	/* Unlocked read; the pool is refilled under the lock below. */
149 	have = min_t(unsigned int, READ_ONCE(pipe->prealloc.count), want);
150 	need = want - have;
151 
152 	if (!need) {
153 		mutex_lock(&pipe->mutex);
154 		return;
155 	}
156 
157 	while (n < need) {
158 		struct page *page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
159 
160 		if (!page)
161 			break;
162 		pages[n++] = page;
163 	}
164 
165 	mutex_lock(&pipe->mutex);
166 	while (n && anon_pipe_prealloc_push(&pipe->prealloc, pages[n - 1]))
167 		n--;
168 
169 	/*
170 	 * Just flush any extra page that got affected by the TOCTOU
171 	 * effect
172 	 */
173 	while (n)
174 		put_page(pages[--n]);
175 }
176 
177 /*
178  * Called with pipe->mutex held. Trim the pool down to PIPE_PREALLOC_KEEP under
179  * the lock, drop it, then free the excess outside the critical section.
180  */
181 static void anon_pipe_trim_and_unlock(struct pipe_inode_info *pipe)
182 {
183 	struct page *excess[PIPE_PREALLOC_MAX];
184 	unsigned int nexcess = 0;
185 
186 	while (pipe->prealloc.count > PIPE_PREALLOC_KEEP)
187 		excess[nexcess++] = anon_pipe_prealloc_pop(&pipe->prealloc);
188 	mutex_unlock(&pipe->mutex);
189 
190 	while (nexcess)
191 		put_page(excess[--nexcess]);
192 }
193 
194 static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe)
195 {
196 	struct page *page;
197 
198 	/* Drain the prealloc pool before allocating. Called with mutex held. */
199 	page = anon_pipe_prealloc_pop(&pipe->prealloc);
200 	if (page)
201 		return page;
202 
203 	/* FWIW: This is called with pipe->mutex held */
204 	return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
205 }
206 
207 static void anon_pipe_put_page(struct pipe_inode_info *pipe,
208 			       struct page *page)
209 {
210 	if (page_count(page) == 1 &&
211 	    anon_pipe_prealloc_push(&pipe->prealloc, page))
212 		return;
213 
214 	put_page(page);
215 }
216 
217 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
218 				  struct pipe_buffer *buf)
219 {
220 	struct page *page = buf->page;
221 
222 	anon_pipe_put_page(pipe, page);
223 }
224 
225 /**
226  * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
227  * @pipe:	the pipe that the buffer belongs to
228  * @buf:	the buffer to get a reference to
229  *
230  * Description:
231  *	This function grabs an extra reference to @buf. It's used in
232  *	the tee() system call, when we duplicate the buffers in one
233  *	pipe into another.
234  */
235 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
236 {
237 	return try_get_page(buf->page);
238 }
239 EXPORT_SYMBOL(generic_pipe_buf_get);
240 
241 /**
242  * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
243  * @pipe:	the pipe that the buffer belongs to
244  * @buf:	the buffer to put a reference to
245  *
246  * Description:
247  *	This function releases a reference to @buf.
248  */
249 void generic_pipe_buf_release(struct pipe_inode_info *pipe,
250 			      struct pipe_buffer *buf)
251 {
252 	put_page(buf->page);
253 }
254 EXPORT_SYMBOL(generic_pipe_buf_release);
255 
256 static const struct pipe_buf_operations anon_pipe_buf_ops = {
257 	.release	= anon_pipe_buf_release,
258 	.get		= generic_pipe_buf_get,
259 };
260 
261 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
262 static inline bool pipe_readable(const struct pipe_inode_info *pipe)
263 {
264 	union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
265 	unsigned int writers = READ_ONCE(pipe->writers);
266 
267 	return !pipe_empty(idx.head, idx.tail) || !writers;
268 }
269 
270 static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,
271 					    struct pipe_buffer *buf,
272 					    unsigned int tail)
273 {
274 	pipe_buf_release(pipe, buf);
275 
276 	/*
277 	 * If the pipe has a watch_queue, we need additional protection
278 	 * by the spinlock because notifications get posted with only
279 	 * this spinlock, no mutex
280 	 */
281 	if (pipe_has_watch_queue(pipe)) {
282 		spin_lock_irq(&pipe->rd_wait.lock);
283 #ifdef CONFIG_WATCH_QUEUE
284 		if (buf->flags & PIPE_BUF_FLAG_LOSS)
285 			pipe->note_loss = true;
286 #endif
287 		pipe->tail = ++tail;
288 		spin_unlock_irq(&pipe->rd_wait.lock);
289 		return tail;
290 	}
291 
292 	/*
293 	 * Without a watch_queue, we can simply increment the tail
294 	 * without the spinlock - the mutex is enough.
295 	 */
296 	pipe->tail = ++tail;
297 	return tail;
298 }
299 
300 static ssize_t
301 anon_pipe_read(struct kiocb *iocb, struct iov_iter *to)
302 {
303 	size_t total_len = iov_iter_count(to);
304 	struct file *filp = iocb->ki_filp;
305 	struct pipe_inode_info *pipe = filp->private_data;
306 	bool wake_writer = false, wake_next_reader = false;
307 	ssize_t ret;
308 
309 	/* Null read succeeds. */
310 	if (unlikely(total_len == 0))
311 		return 0;
312 
313 	ret = 0;
314 	mutex_lock(&pipe->mutex);
315 
316 	/*
317 	 * We only wake up writers if the pipe was full when we started reading
318 	 * and it is no longer full after reading to avoid unnecessary wakeups.
319 	 *
320 	 * But when we do wake up writers, we do so using a sync wakeup
321 	 * (WF_SYNC), because we want them to get going and generate more
322 	 * data for us.
323 	 */
324 	for (;;) {
325 		/* Read ->head with a barrier vs post_one_notification() */
326 		unsigned int head = smp_load_acquire(&pipe->head);
327 		unsigned int tail = pipe->tail;
328 
329 #ifdef CONFIG_WATCH_QUEUE
330 		if (pipe->note_loss) {
331 			struct watch_notification n;
332 
333 			if (total_len < 8) {
334 				if (ret == 0)
335 					ret = -ENOBUFS;
336 				break;
337 			}
338 
339 			n.type = WATCH_TYPE_META;
340 			n.subtype = WATCH_META_LOSS_NOTIFICATION;
341 			n.info = watch_sizeof(n);
342 			if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
343 				if (ret == 0)
344 					ret = -EFAULT;
345 				break;
346 			}
347 			ret += sizeof(n);
348 			total_len -= sizeof(n);
349 			pipe->note_loss = false;
350 		}
351 #endif
352 
353 		if (!pipe_empty(head, tail)) {
354 			struct pipe_buffer *buf = pipe_buf(pipe, tail);
355 			size_t chars = buf->len;
356 			size_t written;
357 			int error;
358 
359 			if (chars > total_len) {
360 				if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
361 					if (ret == 0)
362 						ret = -ENOBUFS;
363 					break;
364 				}
365 				chars = total_len;
366 			}
367 
368 			error = pipe_buf_confirm(pipe, buf);
369 			if (error) {
370 				if (!ret)
371 					ret = error;
372 				break;
373 			}
374 
375 			written = copy_page_to_iter(buf->page, buf->offset, chars, to);
376 			if (unlikely(written < chars)) {
377 				if (!ret)
378 					ret = -EFAULT;
379 				break;
380 			}
381 			ret += chars;
382 			buf->offset += chars;
383 			buf->len -= chars;
384 
385 			/* Was it a packet buffer? Clean up and exit */
386 			if (buf->flags & PIPE_BUF_FLAG_PACKET) {
387 				total_len = chars;
388 				buf->len = 0;
389 			}
390 
391 			if (!buf->len) {
392 				wake_writer |= pipe_full(head, tail, pipe->max_usage);
393 				tail = pipe_update_tail(pipe, buf, tail);
394 			}
395 			total_len -= chars;
396 			if (!total_len)
397 				break;	/* common path: read succeeded */
398 			if (!pipe_empty(head, tail))	/* More to do? */
399 				continue;
400 		}
401 
402 		if (!pipe->writers)
403 			break;
404 		if (ret)
405 			break;
406 		if ((filp->f_flags & O_NONBLOCK) ||
407 		    (iocb->ki_flags & IOCB_NOWAIT)) {
408 			ret = -EAGAIN;
409 			break;
410 		}
411 		mutex_unlock(&pipe->mutex);
412 		/*
413 		 * We only get here if we didn't actually read anything.
414 		 *
415 		 * But because we didn't read anything, at this point we can
416 		 * just return directly with -ERESTARTSYS if we're interrupted,
417 		 * since we've done any required wakeups and there's no need
418 		 * to mark anything accessed. And we've dropped the lock.
419 		 */
420 		if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
421 			return -ERESTARTSYS;
422 
423 		wake_next_reader = true;
424 		mutex_lock(&pipe->mutex);
425 	}
426 	if (pipe_is_empty(pipe))
427 		wake_next_reader = false;
428 	/* Consumed buffers may have refilled the pool; trim it and unlock. */
429 	anon_pipe_trim_and_unlock(pipe);
430 
431 	if (wake_writer)
432 		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
433 	if (wake_next_reader)
434 		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
435 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
436 	return ret;
437 }
438 
439 static ssize_t
440 fifo_pipe_read(struct kiocb *iocb, struct iov_iter *to)
441 {
442 	int ret = anon_pipe_read(iocb, to);
443 	if (ret > 0)
444 		file_accessed(iocb->ki_filp);
445 	return ret;
446 }
447 
448 static inline int is_packetized(struct file *file)
449 {
450 	return (file->f_flags & O_DIRECT) != 0;
451 }
452 
453 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
454 static inline bool pipe_writable(const struct pipe_inode_info *pipe)
455 {
456 	union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
457 	unsigned int max_usage = READ_ONCE(pipe->max_usage);
458 
459 	return !pipe_full(idx.head, idx.tail, max_usage) ||
460 		!READ_ONCE(pipe->readers);
461 }
462 
463 static ssize_t
464 anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
465 {
466 	struct file *filp = iocb->ki_filp;
467 	struct pipe_inode_info *pipe = filp->private_data;
468 	unsigned int head;
469 	ssize_t ret = 0;
470 	size_t total_len = iov_iter_count(from);
471 	ssize_t chars;
472 	bool was_empty = false;
473 	bool wake_next_writer = false;
474 
475 	/*
476 	 * Reject writing to watch queue pipes before the point where we lock
477 	 * the pipe.
478 	 * Otherwise, lockdep would be unhappy if the caller already has another
479 	 * pipe locked.
480 	 * If we had to support locking a normal pipe and a notification pipe at
481 	 * the same time, we could set up lockdep annotations for that, but
482 	 * since we don't actually need that, it's simpler to just bail here.
483 	 */
484 	if (pipe_has_watch_queue(pipe))
485 		return -EXDEV;
486 
487 	/* Null write succeeds. */
488 	if (unlikely(total_len == 0))
489 		return 0;
490 
491 	anon_pipe_prefill_and_lock(pipe, total_len);
492 
493 	if (!pipe->readers) {
494 		if ((iocb->ki_flags & IOCB_NOSIGNAL) == 0)
495 			send_sig(SIGPIPE, current, 0);
496 		ret = -EPIPE;
497 		goto out;
498 	}
499 
500 	/*
501 	 * If it wasn't empty we try to merge new data into
502 	 * the last buffer.
503 	 *
504 	 * That naturally merges small writes, but it also
505 	 * page-aligns the rest of the writes for large writes
506 	 * spanning multiple pages.
507 	 */
508 	head = pipe->head;
509 	was_empty = pipe_empty(head, pipe->tail);
510 	chars = total_len & (PAGE_SIZE-1);
511 	if (chars && !was_empty) {
512 		struct pipe_buffer *buf = pipe_buf(pipe, head - 1);
513 		int offset = buf->offset + buf->len;
514 
515 		if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
516 		    offset + chars <= PAGE_SIZE) {
517 			ret = pipe_buf_confirm(pipe, buf);
518 			if (ret)
519 				goto out;
520 
521 			ret = copy_page_from_iter(buf->page, offset, chars, from);
522 			if (unlikely(ret < chars)) {
523 				ret = -EFAULT;
524 				goto out;
525 			}
526 
527 			buf->len += ret;
528 			if (!iov_iter_count(from))
529 				goto out;
530 		}
531 	}
532 
533 	for (;;) {
534 		if (!pipe->readers) {
535 			if ((iocb->ki_flags & IOCB_NOSIGNAL) == 0)
536 				send_sig(SIGPIPE, current, 0);
537 			if (!ret)
538 				ret = -EPIPE;
539 			break;
540 		}
541 
542 		head = pipe->head;
543 		if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
544 			struct pipe_buffer *buf;
545 			struct page *page;
546 			int copied;
547 
548 			page = anon_pipe_get_page(pipe);
549 			if (unlikely(!page)) {
550 				if (!ret)
551 					ret = -ENOMEM;
552 				break;
553 			}
554 
555 			copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
556 			if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
557 				anon_pipe_put_page(pipe, page);
558 				if (!ret)
559 					ret = -EFAULT;
560 				break;
561 			}
562 
563 			pipe->head = head + 1;
564 			/* Insert it into the buffer array */
565 			buf = pipe_buf(pipe, head);
566 			buf->page = page;
567 			buf->ops = &anon_pipe_buf_ops;
568 			buf->offset = 0;
569 			if (is_packetized(filp))
570 				buf->flags = PIPE_BUF_FLAG_PACKET;
571 			else
572 				buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
573 
574 			buf->len = copied;
575 			ret += copied;
576 
577 			if (!iov_iter_count(from))
578 				break;
579 
580 			continue;
581 		}
582 
583 		/* Wait for buffer space to become available. */
584 		if ((filp->f_flags & O_NONBLOCK) ||
585 		    (iocb->ki_flags & IOCB_NOWAIT)) {
586 			if (!ret)
587 				ret = -EAGAIN;
588 			break;
589 		}
590 		if (signal_pending(current)) {
591 			if (!ret)
592 				ret = -ERESTARTSYS;
593 			break;
594 		}
595 
596 		/*
597 		 * We're going to release the pipe lock and wait for more
598 		 * space. We wake up any readers if necessary, and then
599 		 * after waiting we need to re-check whether the pipe
600 		 * become empty while we dropped the lock.
601 		 */
602 		mutex_unlock(&pipe->mutex);
603 		if (was_empty)
604 			wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
605 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
606 		wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
607 		mutex_lock(&pipe->mutex);
608 		was_empty = pipe_is_empty(pipe);
609 		wake_next_writer = true;
610 	}
611 out:
612 	if (pipe_is_full(pipe))
613 		wake_next_writer = false;
614 	anon_pipe_trim_and_unlock(pipe);
615 
616 	/*
617 	 * If we do do a wakeup event, we do a 'sync' wakeup, because we
618 	 * want the reader to start processing things asap, rather than
619 	 * leave the data pending.
620 	 *
621 	 * This is particularly important for small writes, because of
622 	 * how (for example) the GNU make jobserver uses small writes to
623 	 * wake up pending jobs
624 	 *
625 	 * ->pseudo_edgetrigger enables per-write wakeups, see pipe_poll()
626 	 */
627 	if (was_empty || READ_ONCE(pipe->pseudo_edgetrigger))
628 		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
629 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
630 	if (wake_next_writer)
631 		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
632 	return ret;
633 }
634 
635 static ssize_t
636 fifo_pipe_write(struct kiocb *iocb, struct iov_iter *from)
637 {
638 	int ret = anon_pipe_write(iocb, from);
639 	if (ret > 0) {
640 		struct file *filp = iocb->ki_filp;
641 		if (sb_start_write_trylock(file_inode(filp)->i_sb)) {
642 			int err = file_update_time(filp);
643 			if (err)
644 				ret = err;
645 			sb_end_write(file_inode(filp)->i_sb);
646 		}
647 	}
648 	return ret;
649 }
650 
651 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
652 {
653 	struct pipe_inode_info *pipe = filp->private_data;
654 	unsigned int count, head, tail;
655 
656 	switch (cmd) {
657 	case FIONREAD:
658 		mutex_lock(&pipe->mutex);
659 		count = 0;
660 		head = pipe->head;
661 		tail = pipe->tail;
662 
663 		while (!pipe_empty(head, tail)) {
664 			count += pipe_buf(pipe, tail)->len;
665 			tail++;
666 		}
667 		mutex_unlock(&pipe->mutex);
668 
669 		return put_user(count, (int __user *)arg);
670 
671 #ifdef CONFIG_WATCH_QUEUE
672 	case IOC_WATCH_QUEUE_SET_SIZE: {
673 		int ret;
674 		mutex_lock(&pipe->mutex);
675 		ret = watch_queue_set_size(pipe, arg);
676 		mutex_unlock(&pipe->mutex);
677 		return ret;
678 	}
679 
680 	case IOC_WATCH_QUEUE_SET_FILTER:
681 		return watch_queue_set_filter(
682 			pipe, (struct watch_notification_filter __user *)arg);
683 #endif
684 
685 	default:
686 		return -ENOIOCTLCMD;
687 	}
688 }
689 
690 static __poll_t
691 pipe_poll(struct file *filp, poll_table *wait)
692 {
693 	__poll_t mask;
694 	struct pipe_inode_info *pipe = filp->private_data;
695 	union pipe_index idx;
696 
697 	/*
698 	 * Legacy epoll(EPOLLET) users depend on historical per-write wakeups,
699 	 * see 3a34b13a88ca ("pipe: make pipe writes always wake up readers")
700 	 * and the ->pseudo_edgetrigger check in anon_pipe_write().
701 	 * Currently io_uring sets EPOLLET for multishot polls, so it gets the
702 	 * same behaviour.
703 	 */
704 	if ((filp->f_mode & FMODE_READ) &&
705 	    wait && (wait->_key & EPOLLET) &&
706 	    unlikely(!READ_ONCE(pipe->pseudo_edgetrigger)))
707 		WRITE_ONCE(pipe->pseudo_edgetrigger, true);
708 
709 	/*
710 	 * Reading pipe state only -- no need for acquiring the semaphore.
711 	 *
712 	 * But because this is racy, the code has to add the
713 	 * entry to the poll table _first_ ..
714 	 */
715 	if (filp->f_mode & FMODE_READ)
716 		poll_wait(filp, &pipe->rd_wait, wait);
717 	if (filp->f_mode & FMODE_WRITE)
718 		poll_wait(filp, &pipe->wr_wait, wait);
719 
720 	/*
721 	 * .. and only then can you do the racy tests. That way,
722 	 * if something changes and you got it wrong, the poll
723 	 * table entry will wake you up and fix it.
724 	 */
725 	idx.head_tail = READ_ONCE(pipe->head_tail);
726 
727 	mask = 0;
728 	if (filp->f_mode & FMODE_READ) {
729 		if (!pipe_empty(idx.head, idx.tail))
730 			mask |= EPOLLIN | EPOLLRDNORM;
731 		if (!pipe->writers && filp->f_pipe != pipe->w_counter)
732 			mask |= EPOLLHUP;
733 	}
734 
735 	if (filp->f_mode & FMODE_WRITE) {
736 		if (!pipe_full(idx.head, idx.tail, pipe->max_usage))
737 			mask |= EPOLLOUT | EPOLLWRNORM;
738 		/*
739 		 * Most Unices do not set EPOLLERR for FIFOs but on Linux they
740 		 * behave exactly like pipes for poll().
741 		 */
742 		if (!pipe->readers)
743 			mask |= EPOLLERR;
744 	}
745 
746 	return mask;
747 }
748 
749 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
750 {
751 	int kill = 0;
752 
753 	spin_lock(&inode->i_lock);
754 	if (!--pipe->files) {
755 		inode->i_pipe = NULL;
756 		kill = 1;
757 	}
758 	spin_unlock(&inode->i_lock);
759 
760 	if (kill)
761 		free_pipe_info(pipe);
762 }
763 
764 static int
765 pipe_release(struct inode *inode, struct file *file)
766 {
767 	struct pipe_inode_info *pipe = file->private_data;
768 
769 	mutex_lock(&pipe->mutex);
770 	if (file->f_mode & FMODE_READ)
771 		pipe->readers--;
772 	if (file->f_mode & FMODE_WRITE)
773 		pipe->writers--;
774 
775 	/* Was that the last reader or writer, but not the other side? */
776 	if (!pipe->readers != !pipe->writers) {
777 		wake_up_interruptible_all(&pipe->rd_wait);
778 		wake_up_interruptible_all(&pipe->wr_wait);
779 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
780 		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
781 	}
782 	mutex_unlock(&pipe->mutex);
783 
784 	put_pipe_info(inode, pipe);
785 	return 0;
786 }
787 
788 static int
789 pipe_fasync(int fd, struct file *filp, int on)
790 {
791 	struct pipe_inode_info *pipe = filp->private_data;
792 	int retval = 0;
793 
794 	mutex_lock(&pipe->mutex);
795 	if (filp->f_mode & FMODE_READ)
796 		retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
797 	if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
798 		int read_ret = retval;
799 
800 		retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
801 		/* Undo only a reader entry added by this call. */
802 		if (retval < 0 && read_ret > 0)
803 			fasync_helper(-1, filp, 0, &pipe->fasync_readers);
804 	}
805 	mutex_unlock(&pipe->mutex);
806 	return retval;
807 }
808 
809 unsigned long account_pipe_buffers(struct user_struct *user,
810 				   unsigned long old, unsigned long new)
811 {
812 	return atomic_long_add_return(new - old, &user->pipe_bufs);
813 }
814 
815 bool too_many_pipe_buffers_soft(unsigned long user_bufs)
816 {
817 	unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
818 
819 	return soft_limit && user_bufs > soft_limit;
820 }
821 
822 bool too_many_pipe_buffers_hard(unsigned long user_bufs)
823 {
824 	unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
825 
826 	return hard_limit && user_bufs > hard_limit;
827 }
828 
829 bool pipe_is_unprivileged_user(void)
830 {
831 	return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
832 }
833 
834 struct pipe_inode_info *alloc_pipe_info(void)
835 {
836 	struct pipe_inode_info *pipe;
837 	unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
838 	struct user_struct *user = get_current_user();
839 	unsigned long user_bufs;
840 	unsigned int max_size = READ_ONCE(pipe_max_size);
841 
842 	pipe = kzalloc_obj(struct pipe_inode_info, GFP_KERNEL_ACCOUNT);
843 	if (pipe == NULL)
844 		goto out_free_uid;
845 
846 	if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
847 		pipe_bufs = max_size >> PAGE_SHIFT;
848 
849 	user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
850 
851 	if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
852 		user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
853 		pipe_bufs = PIPE_MIN_DEF_BUFFERS;
854 	}
855 
856 	if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
857 		goto out_revert_acct;
858 
859 	pipe->bufs = kzalloc_objs(struct pipe_buffer, pipe_bufs,
860 				  GFP_KERNEL_ACCOUNT);
861 
862 	if (pipe->bufs) {
863 		init_waitqueue_head(&pipe->rd_wait);
864 		init_waitqueue_head(&pipe->wr_wait);
865 		pipe->r_counter = pipe->w_counter = 1;
866 		pipe->max_usage = pipe_bufs;
867 		pipe->ring_size = pipe_bufs;
868 		pipe->nr_accounted = pipe_bufs;
869 		pipe->user = user;
870 		mutex_init(&pipe->mutex);
871 		lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL);
872 		return pipe;
873 	}
874 
875 out_revert_acct:
876 	(void) account_pipe_buffers(user, pipe_bufs, 0);
877 	kfree(pipe);
878 out_free_uid:
879 	free_uid(user);
880 	return NULL;
881 }
882 
883 void free_pipe_info(struct pipe_inode_info *pipe)
884 {
885 	unsigned int i;
886 
887 #ifdef CONFIG_WATCH_QUEUE
888 	if (pipe->watch_queue)
889 		watch_queue_clear(pipe->watch_queue);
890 #endif
891 
892 	(void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
893 	free_uid(pipe->user);
894 	for (i = 0; i < pipe->ring_size; i++) {
895 		struct pipe_buffer *buf = pipe->bufs + i;
896 		if (buf->ops)
897 			pipe_buf_release(pipe, buf);
898 	}
899 #ifdef CONFIG_WATCH_QUEUE
900 	if (pipe->watch_queue)
901 		put_watch_queue(pipe->watch_queue);
902 #endif
903 	for (i = 0; i < pipe->prealloc.count; i++)
904 		__free_page(pipe->prealloc.pages[i]);
905 	kfree(pipe->bufs);
906 	kfree(pipe);
907 }
908 
909 static struct vfsmount *pipe_mnt __ro_after_init;
910 
911 /*
912  * pipefs_dname() is called from d_path().
913  */
914 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
915 {
916 	return dynamic_dname(buffer, buflen, "pipe:[%llu]",
917 				d_inode(dentry)->i_ino);
918 }
919 
920 static const struct dentry_operations pipefs_dentry_operations = {
921 	.d_dname	= pipefs_dname,
922 };
923 
924 static const struct file_operations pipeanon_fops;
925 
926 static struct inode * get_pipe_inode(void)
927 {
928 	struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
929 	struct pipe_inode_info *pipe;
930 
931 	if (!inode)
932 		goto fail_inode;
933 
934 	inode->i_ino = get_next_ino();
935 
936 	pipe = alloc_pipe_info();
937 	if (!pipe)
938 		goto fail_iput;
939 
940 	inode->i_pipe = pipe;
941 	pipe->files = 2;
942 	pipe->readers = pipe->writers = 1;
943 	inode->i_fop = &pipeanon_fops;
944 
945 	/*
946 	 * Mark the inode dirty from the very beginning,
947 	 * that way it will never be moved to the dirty
948 	 * list because "mark_inode_dirty()" will think
949 	 * that it already _is_ on the dirty list.
950 	 */
951 	inode_state_assign_raw(inode, I_DIRTY);
952 	inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
953 	inode->i_uid = current_fsuid();
954 	inode->i_gid = current_fsgid();
955 	simple_inode_init_ts(inode);
956 
957 	return inode;
958 
959 fail_iput:
960 	iput(inode);
961 
962 fail_inode:
963 	return NULL;
964 }
965 
966 int create_pipe_files(struct file **res, int flags)
967 {
968 	struct inode *inode = get_pipe_inode();
969 	struct file *f;
970 	int error;
971 
972 	if (!inode)
973 		return -ENFILE;
974 
975 	if (flags & O_NOTIFICATION_PIPE) {
976 		error = watch_queue_init(inode->i_pipe);
977 		if (error) {
978 			free_pipe_info(inode->i_pipe);
979 			iput(inode);
980 			return error;
981 		}
982 	}
983 
984 	f = alloc_file_pseudo(inode, pipe_mnt, "",
985 				O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
986 				&pipeanon_fops);
987 	if (IS_ERR(f)) {
988 		free_pipe_info(inode->i_pipe);
989 		iput(inode);
990 		return PTR_ERR(f);
991 	}
992 
993 	f->private_data = inode->i_pipe;
994 	f->f_pipe = 0;
995 
996 	res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),
997 				  &pipeanon_fops);
998 	if (IS_ERR(res[0])) {
999 		put_pipe_info(inode, inode->i_pipe);
1000 		fput(f);
1001 		return PTR_ERR(res[0]);
1002 	}
1003 	res[0]->private_data = inode->i_pipe;
1004 	res[0]->f_pipe = 0;
1005 	res[1] = f;
1006 	stream_open(inode, res[0]);
1007 	stream_open(inode, res[1]);
1008 
1009 	/* pipe groks IOCB_NOWAIT */
1010 	res[0]->f_mode |= FMODE_NOWAIT;
1011 	res[1]->f_mode |= FMODE_NOWAIT;
1012 
1013 	/*
1014 	 * Disable permission and pre-content events, but enable legacy
1015 	 * inotify events for legacy users.
1016 	 */
1017 	file_set_fsnotify_mode(res[0], FMODE_NONOTIFY_PERM);
1018 	file_set_fsnotify_mode(res[1], FMODE_NONOTIFY_PERM);
1019 	return 0;
1020 }
1021 
1022 static int __do_pipe_flags(int *fd, struct file **files, int flags)
1023 {
1024 	int error;
1025 	int fdw, fdr;
1026 
1027 	if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
1028 		return -EINVAL;
1029 
1030 	error = create_pipe_files(files, flags);
1031 	if (error)
1032 		return error;
1033 
1034 	error = get_unused_fd_flags(flags);
1035 	if (error < 0)
1036 		goto err_read_pipe;
1037 	fdr = error;
1038 
1039 	error = get_unused_fd_flags(flags);
1040 	if (error < 0)
1041 		goto err_fdr;
1042 	fdw = error;
1043 
1044 	audit_fd_pair(fdr, fdw);
1045 	fd[0] = fdr;
1046 	fd[1] = fdw;
1047 	return 0;
1048 
1049  err_fdr:
1050 	put_unused_fd(fdr);
1051  err_read_pipe:
1052 	fput(files[0]);
1053 	fput(files[1]);
1054 	return error;
1055 }
1056 
1057 int do_pipe_flags(int *fd, int flags)
1058 {
1059 	struct file *files[2];
1060 	int error = __do_pipe_flags(fd, files, flags);
1061 	if (!error) {
1062 		fd_install(fd[0], files[0]);
1063 		fd_install(fd[1], files[1]);
1064 	}
1065 	return error;
1066 }
1067 
1068 /*
1069  * sys_pipe() is the normal C calling standard for creating
1070  * a pipe. It's not the way Unix traditionally does this, though.
1071  */
1072 static int do_pipe2(int __user *fildes, int flags)
1073 {
1074 	struct file *files[2];
1075 	int fd[2];
1076 	int error;
1077 
1078 	error = __do_pipe_flags(fd, files, flags);
1079 	if (!error) {
1080 		if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
1081 			fput(files[0]);
1082 			fput(files[1]);
1083 			put_unused_fd(fd[0]);
1084 			put_unused_fd(fd[1]);
1085 			error = -EFAULT;
1086 		} else {
1087 			fd_install(fd[0], files[0]);
1088 			fd_install(fd[1], files[1]);
1089 		}
1090 	}
1091 	return error;
1092 }
1093 
1094 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
1095 {
1096 	return do_pipe2(fildes, flags);
1097 }
1098 
1099 SYSCALL_DEFINE1(pipe, int __user *, fildes)
1100 {
1101 	return do_pipe2(fildes, 0);
1102 }
1103 
1104 /*
1105  * This is the stupid "wait for pipe to be readable or writable"
1106  * model.
1107  *
1108  * See pipe_read/write() for the proper kind of exclusive wait,
1109  * but that requires that we wake up any other readers/writers
1110  * if we then do not end up reading everything (ie the whole
1111  * "wake_next_reader/writer" logic in pipe_read/write()).
1112  */
1113 void pipe_wait_readable(struct pipe_inode_info *pipe)
1114 {
1115 	pipe_unlock(pipe);
1116 	wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
1117 	pipe_lock(pipe);
1118 }
1119 
1120 void pipe_wait_writable(struct pipe_inode_info *pipe)
1121 {
1122 	pipe_unlock(pipe);
1123 	wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
1124 	pipe_lock(pipe);
1125 }
1126 
1127 /*
1128  * This depends on both the wait (here) and the wakeup (wake_up_partner)
1129  * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1130  * race with the count check and waitqueue prep.
1131  *
1132  * Normally in order to avoid races, you'd do the prepare_to_wait() first,
1133  * then check the condition you're waiting for, and only then sleep. But
1134  * because of the pipe lock, we can check the condition before being on
1135  * the wait queue.
1136  *
1137  * We use the 'rd_wait' waitqueue for pipe partner waiting.
1138  */
1139 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
1140 {
1141 	DEFINE_WAIT(rdwait);
1142 	int cur = *cnt;
1143 
1144 	while (cur == *cnt) {
1145 		prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
1146 		pipe_unlock(pipe);
1147 		schedule();
1148 		finish_wait(&pipe->rd_wait, &rdwait);
1149 		pipe_lock(pipe);
1150 		if (signal_pending(current))
1151 			break;
1152 	}
1153 	return cur == *cnt ? -ERESTARTSYS : 0;
1154 }
1155 
1156 static void wake_up_partner(struct pipe_inode_info *pipe)
1157 {
1158 	wake_up_interruptible_all(&pipe->rd_wait);
1159 }
1160 
1161 static int fifo_open(struct inode *inode, struct file *filp)
1162 {
1163 	bool is_pipe = inode->i_fop == &pipeanon_fops;
1164 	struct pipe_inode_info *pipe;
1165 	int ret;
1166 
1167 	filp->f_pipe = 0;
1168 
1169 	spin_lock(&inode->i_lock);
1170 	if (inode->i_pipe) {
1171 		pipe = inode->i_pipe;
1172 		pipe->files++;
1173 		spin_unlock(&inode->i_lock);
1174 	} else {
1175 		spin_unlock(&inode->i_lock);
1176 		pipe = alloc_pipe_info();
1177 		if (!pipe)
1178 			return -ENOMEM;
1179 		pipe->files = 1;
1180 		spin_lock(&inode->i_lock);
1181 		if (unlikely(inode->i_pipe)) {
1182 			inode->i_pipe->files++;
1183 			spin_unlock(&inode->i_lock);
1184 			free_pipe_info(pipe);
1185 			pipe = inode->i_pipe;
1186 		} else {
1187 			inode->i_pipe = pipe;
1188 			spin_unlock(&inode->i_lock);
1189 		}
1190 	}
1191 	filp->private_data = pipe;
1192 	/* OK, we have a pipe and it's pinned down */
1193 
1194 	mutex_lock(&pipe->mutex);
1195 
1196 	/* We can only do regular read/write on fifos */
1197 	stream_open(inode, filp);
1198 
1199 	switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {
1200 	case FMODE_READ:
1201 	/*
1202 	 *  O_RDONLY
1203 	 *  POSIX.1 says that O_NONBLOCK means return with the FIFO
1204 	 *  opened, even when there is no process writing the FIFO.
1205 	 */
1206 		pipe->r_counter++;
1207 		if (pipe->readers++ == 0)
1208 			wake_up_partner(pipe);
1209 
1210 		if (!is_pipe && !pipe->writers) {
1211 			if ((filp->f_flags & O_NONBLOCK)) {
1212 				/* suppress EPOLLHUP until we have
1213 				 * seen a writer */
1214 				filp->f_pipe = pipe->w_counter;
1215 			} else {
1216 				if (wait_for_partner(pipe, &pipe->w_counter))
1217 					goto err_rd;
1218 			}
1219 		}
1220 		break;
1221 
1222 	case FMODE_WRITE:
1223 	/*
1224 	 *  O_WRONLY
1225 	 *  POSIX.1 says that O_NONBLOCK means return -1 with
1226 	 *  errno=ENXIO when there is no process reading the FIFO.
1227 	 */
1228 		ret = -ENXIO;
1229 		if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
1230 			goto err;
1231 
1232 		pipe->w_counter++;
1233 		if (!pipe->writers++)
1234 			wake_up_partner(pipe);
1235 
1236 		if (!is_pipe && !pipe->readers) {
1237 			if (wait_for_partner(pipe, &pipe->r_counter))
1238 				goto err_wr;
1239 		}
1240 		break;
1241 
1242 	case FMODE_READ | FMODE_WRITE:
1243 	/*
1244 	 *  O_RDWR
1245 	 *  POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
1246 	 *  This implementation will NEVER block on a O_RDWR open, since
1247 	 *  the process can at least talk to itself.
1248 	 */
1249 
1250 		pipe->readers++;
1251 		pipe->writers++;
1252 		pipe->r_counter++;
1253 		pipe->w_counter++;
1254 		if (pipe->readers == 1 || pipe->writers == 1)
1255 			wake_up_partner(pipe);
1256 		break;
1257 
1258 	default:
1259 		ret = -EINVAL;
1260 		goto err;
1261 	}
1262 
1263 	/* Ok! */
1264 	mutex_unlock(&pipe->mutex);
1265 	return 0;
1266 
1267 err_rd:
1268 	if (!--pipe->readers)
1269 		wake_up_interruptible(&pipe->wr_wait);
1270 	ret = -ERESTARTSYS;
1271 	goto err;
1272 
1273 err_wr:
1274 	if (!--pipe->writers)
1275 		wake_up_interruptible_all(&pipe->rd_wait);
1276 	ret = -ERESTARTSYS;
1277 	goto err;
1278 
1279 err:
1280 	mutex_unlock(&pipe->mutex);
1281 
1282 	put_pipe_info(inode, pipe);
1283 	return ret;
1284 }
1285 
1286 const struct file_operations pipefifo_fops = {
1287 	.open		= fifo_open,
1288 	.read_iter	= fifo_pipe_read,
1289 	.write_iter	= fifo_pipe_write,
1290 	.poll		= pipe_poll,
1291 	.unlocked_ioctl	= pipe_ioctl,
1292 	.release	= pipe_release,
1293 	.fasync		= pipe_fasync,
1294 	.splice_write	= iter_file_splice_write,
1295 };
1296 
1297 static const struct file_operations pipeanon_fops = {
1298 	.open		= fifo_open,
1299 	.read_iter	= anon_pipe_read,
1300 	.write_iter	= anon_pipe_write,
1301 	.poll		= pipe_poll,
1302 	.unlocked_ioctl	= pipe_ioctl,
1303 	.release	= pipe_release,
1304 	.fasync		= pipe_fasync,
1305 	.splice_write	= iter_file_splice_write,
1306 };
1307 
1308 /*
1309  * Currently we rely on the pipe array holding a power-of-2 number
1310  * of pages. Returns 0 on error.
1311  */
1312 unsigned int round_pipe_size(unsigned int size)
1313 {
1314 	if (size > (1U << 31))
1315 		return 0;
1316 
1317 	/* Minimum pipe size, as required by POSIX */
1318 	if (size < PAGE_SIZE)
1319 		return PAGE_SIZE;
1320 
1321 	return roundup_pow_of_two(size);
1322 }
1323 
1324 /*
1325  * Resize the pipe ring to a number of slots.
1326  *
1327  * Note the pipe can be reduced in capacity, but only if the current
1328  * occupancy doesn't exceed nr_slots; if it does, EBUSY will be
1329  * returned instead.
1330  */
1331 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
1332 {
1333 	struct pipe_buffer *bufs;
1334 	unsigned int head, tail, mask, n;
1335 
1336 	/* nr_slots larger than limits of pipe->{head,tail} */
1337 	if (unlikely(nr_slots > (pipe_index_t)-1u))
1338 		return -EINVAL;
1339 
1340 	bufs = kzalloc_objs(*bufs, nr_slots, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
1341 	if (unlikely(!bufs))
1342 		return -ENOMEM;
1343 
1344 	spin_lock_irq(&pipe->rd_wait.lock);
1345 	mask = pipe->ring_size - 1;
1346 	head = pipe->head;
1347 	tail = pipe->tail;
1348 
1349 	n = pipe_occupancy(head, tail);
1350 	if (nr_slots < n) {
1351 		spin_unlock_irq(&pipe->rd_wait.lock);
1352 		kfree(bufs);
1353 		return -EBUSY;
1354 	}
1355 
1356 	/*
1357 	 * The pipe array wraps around, so just start the new one at zero
1358 	 * and adjust the indices.
1359 	 */
1360 	if (n > 0) {
1361 		unsigned int h = head & mask;
1362 		unsigned int t = tail & mask;
1363 		if (h > t) {
1364 			memcpy(bufs, pipe->bufs + t,
1365 			       n * sizeof(struct pipe_buffer));
1366 		} else {
1367 			unsigned int tsize = pipe->ring_size - t;
1368 			if (h > 0)
1369 				memcpy(bufs + tsize, pipe->bufs,
1370 				       h * sizeof(struct pipe_buffer));
1371 			memcpy(bufs, pipe->bufs + t,
1372 			       tsize * sizeof(struct pipe_buffer));
1373 		}
1374 	}
1375 
1376 	head = n;
1377 	tail = 0;
1378 
1379 	kfree(pipe->bufs);
1380 	pipe->bufs = bufs;
1381 	pipe->ring_size = nr_slots;
1382 	if (pipe->max_usage > nr_slots)
1383 		pipe->max_usage = nr_slots;
1384 	pipe->tail = tail;
1385 	pipe->head = head;
1386 
1387 	if (!pipe_has_watch_queue(pipe)) {
1388 		pipe->max_usage = nr_slots;
1389 		pipe->nr_accounted = nr_slots;
1390 	}
1391 
1392 	spin_unlock_irq(&pipe->rd_wait.lock);
1393 
1394 	/* This might have made more room for writers */
1395 	wake_up_interruptible_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
1396 	return 0;
1397 }
1398 
1399 /*
1400  * Allocate a new array of pipe buffers and copy the info over. Returns the
1401  * pipe size if successful, or return -ERROR on error.
1402  */
1403 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg)
1404 {
1405 	unsigned long user_bufs;
1406 	unsigned int nr_slots, size;
1407 	long ret = 0;
1408 
1409 	if (pipe_has_watch_queue(pipe))
1410 		return -EBUSY;
1411 
1412 	size = round_pipe_size(arg);
1413 	nr_slots = size >> PAGE_SHIFT;
1414 
1415 	if (!nr_slots)
1416 		return -EINVAL;
1417 
1418 	/*
1419 	 * If trying to increase the pipe capacity, check that an
1420 	 * unprivileged user is not trying to exceed various limits
1421 	 * (soft limit check here, hard limit check just below).
1422 	 * Decreasing the pipe capacity is always permitted, even
1423 	 * if the user is currently over a limit.
1424 	 */
1425 	if (nr_slots > pipe->max_usage &&
1426 			size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
1427 		return -EPERM;
1428 
1429 	user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
1430 
1431 	if (nr_slots > pipe->max_usage &&
1432 			(too_many_pipe_buffers_hard(user_bufs) ||
1433 			 too_many_pipe_buffers_soft(user_bufs)) &&
1434 			pipe_is_unprivileged_user()) {
1435 		ret = -EPERM;
1436 		goto out_revert_acct;
1437 	}
1438 
1439 	ret = pipe_resize_ring(pipe, nr_slots);
1440 	if (ret < 0)
1441 		goto out_revert_acct;
1442 
1443 	return pipe->max_usage * PAGE_SIZE;
1444 
1445 out_revert_acct:
1446 	(void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
1447 	return ret;
1448 }
1449 
1450 /*
1451  * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is
1452  * not enough to verify that this is a pipe.
1453  */
1454 struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
1455 {
1456 	struct pipe_inode_info *pipe = file->private_data;
1457 
1458 	if (!pipe)
1459 		return NULL;
1460 	if (file->f_op != &pipefifo_fops && file->f_op != &pipeanon_fops)
1461 		return NULL;
1462 	if (for_splice && pipe_has_watch_queue(pipe))
1463 		return NULL;
1464 	return pipe;
1465 }
1466 
1467 long pipe_fcntl(struct file *file, unsigned int cmd, unsigned int arg)
1468 {
1469 	struct pipe_inode_info *pipe;
1470 	long ret;
1471 
1472 	pipe = get_pipe_info(file, false);
1473 	if (!pipe)
1474 		return -EBADF;
1475 
1476 	mutex_lock(&pipe->mutex);
1477 
1478 	switch (cmd) {
1479 	case F_SETPIPE_SZ:
1480 		ret = pipe_set_size(pipe, arg);
1481 		break;
1482 	case F_GETPIPE_SZ:
1483 		ret = pipe->max_usage * PAGE_SIZE;
1484 		break;
1485 	default:
1486 		ret = -EINVAL;
1487 		break;
1488 	}
1489 
1490 	mutex_unlock(&pipe->mutex);
1491 	return ret;
1492 }
1493 
1494 static const struct super_operations pipefs_ops = {
1495 	.destroy_inode = free_inode_nonrcu,
1496 	.statfs = simple_statfs,
1497 };
1498 
1499 /*
1500  * pipefs should _never_ be mounted by userland - too much of security hassle,
1501  * no real gain from having the whole file system mounted. So we don't need
1502  * any operations on the root directory. However, we need a non-trivial
1503  * d_name - pipe: will go nicely and kill the special-casing in procfs.
1504  */
1505 
1506 static int pipefs_init_fs_context(struct fs_context *fc)
1507 {
1508 	struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
1509 	if (!ctx)
1510 		return -ENOMEM;
1511 	ctx->ops = &pipefs_ops;
1512 	ctx->dops = &pipefs_dentry_operations;
1513 	return 0;
1514 }
1515 
1516 static struct file_system_type pipe_fs_type = {
1517 	.name		= "pipefs",
1518 	.init_fs_context = pipefs_init_fs_context,
1519 	.kill_sb	= kill_anon_super,
1520 };
1521 
1522 #ifdef CONFIG_SYSCTL
1523 
1524 static ulong round_pipe_size_ul(ulong size)
1525 {
1526 	return round_pipe_size(size);
1527 }
1528 
1529 static int u2k_pipe_maxsz(const ulong *u_ptr, uint *k_ptr)
1530 {
1531 	return proc_uint_u2k_conv_uop(u_ptr, k_ptr, round_pipe_size_ul);
1532 }
1533 
1534 static int do_proc_uint_conv_pipe_maxsz(bool *negp, ulong *u_ptr, uint *k_ptr,
1535 					int dir, const struct ctl_table *table)
1536 {
1537 	return proc_uint_conv(u_ptr, k_ptr, dir, table, true,
1538 			      u2k_pipe_maxsz,
1539 			      proc_uint_k2u_conv);
1540 }
1541 
1542 static int proc_dopipe_max_size(const struct ctl_table *table, int write,
1543 				void *buffer, size_t *lenp, loff_t *ppos)
1544 {
1545 	return proc_douintvec_conv(table, write, buffer, lenp, ppos,
1546 				   do_proc_uint_conv_pipe_maxsz);
1547 }
1548 
1549 static const struct ctl_table fs_pipe_sysctls[] = {
1550 	{
1551 		.procname	= "pipe-max-size",
1552 		.data		= &pipe_max_size,
1553 		.maxlen		= sizeof(pipe_max_size),
1554 		.mode		= 0644,
1555 		.proc_handler	= proc_dopipe_max_size,
1556 		.extra1		= SYSCTL_ONE,
1557 	},
1558 	{
1559 		.procname	= "pipe-user-pages-hard",
1560 		.data		= &pipe_user_pages_hard,
1561 		.maxlen		= sizeof(pipe_user_pages_hard),
1562 		.mode		= 0644,
1563 		.proc_handler	= proc_doulongvec_minmax,
1564 	},
1565 	{
1566 		.procname	= "pipe-user-pages-soft",
1567 		.data		= &pipe_user_pages_soft,
1568 		.maxlen		= sizeof(pipe_user_pages_soft),
1569 		.mode		= 0644,
1570 		.proc_handler	= proc_doulongvec_minmax,
1571 	},
1572 };
1573 #endif
1574 
1575 static int __init init_pipe_fs(void)
1576 {
1577 	int err = register_filesystem(&pipe_fs_type);
1578 
1579 	if (!err) {
1580 		pipe_mnt = kern_mount(&pipe_fs_type);
1581 		if (IS_ERR(pipe_mnt)) {
1582 			err = PTR_ERR(pipe_mnt);
1583 			unregister_filesystem(&pipe_fs_type);
1584 		}
1585 	}
1586 #ifdef CONFIG_SYSCTL
1587 	register_sysctl_init("fs", fs_pipe_sysctls);
1588 #endif
1589 	return err;
1590 }
1591 
1592 fs_initcall(init_pipe_fs);
1593