xref: /linux/fs/pipe.c (revision c924c5e9b8c65b3a479a90e5e37d74cc8cd9fe0a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/fs/pipe.c
4  *
5  *  Copyright (C) 1991, 1992, 1999  Linus Torvalds
6  */
7 
8 #include <linux/mm.h>
9 #include <linux/file.h>
10 #include <linux/poll.h>
11 #include <linux/slab.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/fs.h>
15 #include <linux/log2.h>
16 #include <linux/mount.h>
17 #include <linux/pseudo_fs.h>
18 #include <linux/magic.h>
19 #include <linux/pipe_fs_i.h>
20 #include <linux/uio.h>
21 #include <linux/highmem.h>
22 #include <linux/pagemap.h>
23 #include <linux/audit.h>
24 #include <linux/syscalls.h>
25 #include <linux/fcntl.h>
26 #include <linux/memcontrol.h>
27 #include <linux/watch_queue.h>
28 #include <linux/sysctl.h>
29 
30 #include <linux/uaccess.h>
31 #include <asm/ioctls.h>
32 
33 #include "internal.h"
34 
35 /*
36  * New pipe buffers will be restricted to this size while the user is exceeding
37  * their pipe buffer quota. The general pipe use case needs at least two
38  * buffers: one for data yet to be read, and one for new data. If this is less
39  * than two, then a write to a non-empty pipe may block even if the pipe is not
40  * full. This can occur with GNU make jobserver or similar uses of pipes as
41  * semaphores: multiple processes may be waiting to write tokens back to the
42  * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
43  *
44  * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
45  * own risk, namely: pipe writes to non-full pipes may block until the pipe is
46  * emptied.
47  */
48 #define PIPE_MIN_DEF_BUFFERS 2
49 
50 /*
51  * The max size that a non-root user is allowed to grow the pipe. Can
52  * be set by root in /proc/sys/fs/pipe-max-size
53  */
54 static unsigned int pipe_max_size = 1048576;
55 
56 /* Maximum allocatable pages per user. Hard limit is unset by default, soft
57  * matches default values.
58  */
59 static unsigned long pipe_user_pages_hard;
60 static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
61 
62 /*
63  * We use head and tail indices that aren't masked off, except at the point of
64  * dereference, but rather they're allowed to wrap naturally.  This means there
65  * isn't a dead spot in the buffer, but the ring has to be a power of two and
66  * <= 2^31.
67  * -- David Howells 2019-09-23.
68  *
69  * Reads with count = 0 should always return 0.
70  * -- Julian Bradfield 1999-06-07.
71  *
72  * FIFOs and Pipes now generate SIGIO for both readers and writers.
73  * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
74  *
75  * pipe_read & write cleanup
76  * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
77  */
78 
79 #define cmp_int(l, r)		((l > r) - (l < r))
80 
81 #ifdef CONFIG_PROVE_LOCKING
pipe_lock_cmp_fn(const struct lockdep_map * a,const struct lockdep_map * b)82 static int pipe_lock_cmp_fn(const struct lockdep_map *a,
83 			    const struct lockdep_map *b)
84 {
85 	return cmp_int((unsigned long) a, (unsigned long) b);
86 }
87 #endif
88 
pipe_lock(struct pipe_inode_info * pipe)89 void pipe_lock(struct pipe_inode_info *pipe)
90 {
91 	if (pipe->files)
92 		mutex_lock(&pipe->mutex);
93 }
94 EXPORT_SYMBOL(pipe_lock);
95 
pipe_unlock(struct pipe_inode_info * pipe)96 void pipe_unlock(struct pipe_inode_info *pipe)
97 {
98 	if (pipe->files)
99 		mutex_unlock(&pipe->mutex);
100 }
101 EXPORT_SYMBOL(pipe_unlock);
102 
pipe_double_lock(struct pipe_inode_info * pipe1,struct pipe_inode_info * pipe2)103 void pipe_double_lock(struct pipe_inode_info *pipe1,
104 		      struct pipe_inode_info *pipe2)
105 {
106 	BUG_ON(pipe1 == pipe2);
107 
108 	if (pipe1 > pipe2)
109 		swap(pipe1, pipe2);
110 
111 	pipe_lock(pipe1);
112 	pipe_lock(pipe2);
113 }
114 
anon_pipe_get_page(struct pipe_inode_info * pipe)115 static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe)
116 {
117 	for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
118 		if (pipe->tmp_page[i]) {
119 			struct page *page = pipe->tmp_page[i];
120 			pipe->tmp_page[i] = NULL;
121 			return page;
122 		}
123 	}
124 
125 	return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
126 }
127 
anon_pipe_put_page(struct pipe_inode_info * pipe,struct page * page)128 static void anon_pipe_put_page(struct pipe_inode_info *pipe,
129 			       struct page *page)
130 {
131 	if (page_count(page) == 1) {
132 		for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
133 			if (!pipe->tmp_page[i]) {
134 				pipe->tmp_page[i] = page;
135 				return;
136 			}
137 		}
138 	}
139 
140 	put_page(page);
141 }
142 
anon_pipe_buf_release(struct pipe_inode_info * pipe,struct pipe_buffer * buf)143 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
144 				  struct pipe_buffer *buf)
145 {
146 	struct page *page = buf->page;
147 
148 	anon_pipe_put_page(pipe, page);
149 }
150 
anon_pipe_buf_try_steal(struct pipe_inode_info * pipe,struct pipe_buffer * buf)151 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
152 		struct pipe_buffer *buf)
153 {
154 	struct page *page = buf->page;
155 
156 	if (page_count(page) != 1)
157 		return false;
158 	memcg_kmem_uncharge_page(page, 0);
159 	__SetPageLocked(page);
160 	return true;
161 }
162 
163 /**
164  * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
165  * @pipe:	the pipe that the buffer belongs to
166  * @buf:	the buffer to attempt to steal
167  *
168  * Description:
169  *	This function attempts to steal the &struct page attached to
170  *	@buf. If successful, this function returns 0 and returns with
171  *	the page locked. The caller may then reuse the page for whatever
172  *	he wishes; the typical use is insertion into a different file
173  *	page cache.
174  */
generic_pipe_buf_try_steal(struct pipe_inode_info * pipe,struct pipe_buffer * buf)175 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
176 		struct pipe_buffer *buf)
177 {
178 	struct page *page = buf->page;
179 
180 	/*
181 	 * A reference of one is golden, that means that the owner of this
182 	 * page is the only one holding a reference to it. lock the page
183 	 * and return OK.
184 	 */
185 	if (page_count(page) == 1) {
186 		lock_page(page);
187 		return true;
188 	}
189 	return false;
190 }
191 EXPORT_SYMBOL(generic_pipe_buf_try_steal);
192 
193 /**
194  * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
195  * @pipe:	the pipe that the buffer belongs to
196  * @buf:	the buffer to get a reference to
197  *
198  * Description:
199  *	This function grabs an extra reference to @buf. It's used in
200  *	the tee() system call, when we duplicate the buffers in one
201  *	pipe into another.
202  */
generic_pipe_buf_get(struct pipe_inode_info * pipe,struct pipe_buffer * buf)203 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
204 {
205 	return try_get_page(buf->page);
206 }
207 EXPORT_SYMBOL(generic_pipe_buf_get);
208 
209 /**
210  * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
211  * @pipe:	the pipe that the buffer belongs to
212  * @buf:	the buffer to put a reference to
213  *
214  * Description:
215  *	This function releases a reference to @buf.
216  */
generic_pipe_buf_release(struct pipe_inode_info * pipe,struct pipe_buffer * buf)217 void generic_pipe_buf_release(struct pipe_inode_info *pipe,
218 			      struct pipe_buffer *buf)
219 {
220 	put_page(buf->page);
221 }
222 EXPORT_SYMBOL(generic_pipe_buf_release);
223 
224 static const struct pipe_buf_operations anon_pipe_buf_ops = {
225 	.release	= anon_pipe_buf_release,
226 	.try_steal	= anon_pipe_buf_try_steal,
227 	.get		= generic_pipe_buf_get,
228 };
229 
230 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
pipe_readable(const struct pipe_inode_info * pipe)231 static inline bool pipe_readable(const struct pipe_inode_info *pipe)
232 {
233 	union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
234 	unsigned int writers = READ_ONCE(pipe->writers);
235 
236 	return !pipe_empty(idx.head, idx.tail) || !writers;
237 }
238 
pipe_update_tail(struct pipe_inode_info * pipe,struct pipe_buffer * buf,unsigned int tail)239 static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,
240 					    struct pipe_buffer *buf,
241 					    unsigned int tail)
242 {
243 	pipe_buf_release(pipe, buf);
244 
245 	/*
246 	 * If the pipe has a watch_queue, we need additional protection
247 	 * by the spinlock because notifications get posted with only
248 	 * this spinlock, no mutex
249 	 */
250 	if (pipe_has_watch_queue(pipe)) {
251 		spin_lock_irq(&pipe->rd_wait.lock);
252 #ifdef CONFIG_WATCH_QUEUE
253 		if (buf->flags & PIPE_BUF_FLAG_LOSS)
254 			pipe->note_loss = true;
255 #endif
256 		pipe->tail = ++tail;
257 		spin_unlock_irq(&pipe->rd_wait.lock);
258 		return tail;
259 	}
260 
261 	/*
262 	 * Without a watch_queue, we can simply increment the tail
263 	 * without the spinlock - the mutex is enough.
264 	 */
265 	pipe->tail = ++tail;
266 	return tail;
267 }
268 
269 static ssize_t
anon_pipe_read(struct kiocb * iocb,struct iov_iter * to)270 anon_pipe_read(struct kiocb *iocb, struct iov_iter *to)
271 {
272 	size_t total_len = iov_iter_count(to);
273 	struct file *filp = iocb->ki_filp;
274 	struct pipe_inode_info *pipe = filp->private_data;
275 	bool wake_writer = false, wake_next_reader = false;
276 	ssize_t ret;
277 
278 	/* Null read succeeds. */
279 	if (unlikely(total_len == 0))
280 		return 0;
281 
282 	ret = 0;
283 	mutex_lock(&pipe->mutex);
284 
285 	/*
286 	 * We only wake up writers if the pipe was full when we started reading
287 	 * and it is no longer full after reading to avoid unnecessary wakeups.
288 	 *
289 	 * But when we do wake up writers, we do so using a sync wakeup
290 	 * (WF_SYNC), because we want them to get going and generate more
291 	 * data for us.
292 	 */
293 	for (;;) {
294 		/* Read ->head with a barrier vs post_one_notification() */
295 		unsigned int head = smp_load_acquire(&pipe->head);
296 		unsigned int tail = pipe->tail;
297 
298 #ifdef CONFIG_WATCH_QUEUE
299 		if (pipe->note_loss) {
300 			struct watch_notification n;
301 
302 			if (total_len < 8) {
303 				if (ret == 0)
304 					ret = -ENOBUFS;
305 				break;
306 			}
307 
308 			n.type = WATCH_TYPE_META;
309 			n.subtype = WATCH_META_LOSS_NOTIFICATION;
310 			n.info = watch_sizeof(n);
311 			if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
312 				if (ret == 0)
313 					ret = -EFAULT;
314 				break;
315 			}
316 			ret += sizeof(n);
317 			total_len -= sizeof(n);
318 			pipe->note_loss = false;
319 		}
320 #endif
321 
322 		if (!pipe_empty(head, tail)) {
323 			struct pipe_buffer *buf = pipe_buf(pipe, tail);
324 			size_t chars = buf->len;
325 			size_t written;
326 			int error;
327 
328 			if (chars > total_len) {
329 				if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
330 					if (ret == 0)
331 						ret = -ENOBUFS;
332 					break;
333 				}
334 				chars = total_len;
335 			}
336 
337 			error = pipe_buf_confirm(pipe, buf);
338 			if (error) {
339 				if (!ret)
340 					ret = error;
341 				break;
342 			}
343 
344 			written = copy_page_to_iter(buf->page, buf->offset, chars, to);
345 			if (unlikely(written < chars)) {
346 				if (!ret)
347 					ret = -EFAULT;
348 				break;
349 			}
350 			ret += chars;
351 			buf->offset += chars;
352 			buf->len -= chars;
353 
354 			/* Was it a packet buffer? Clean up and exit */
355 			if (buf->flags & PIPE_BUF_FLAG_PACKET) {
356 				total_len = chars;
357 				buf->len = 0;
358 			}
359 
360 			if (!buf->len) {
361 				wake_writer |= pipe_full(head, tail, pipe->max_usage);
362 				tail = pipe_update_tail(pipe, buf, tail);
363 			}
364 			total_len -= chars;
365 			if (!total_len)
366 				break;	/* common path: read succeeded */
367 			if (!pipe_empty(head, tail))	/* More to do? */
368 				continue;
369 		}
370 
371 		if (!pipe->writers)
372 			break;
373 		if (ret)
374 			break;
375 		if ((filp->f_flags & O_NONBLOCK) ||
376 		    (iocb->ki_flags & IOCB_NOWAIT)) {
377 			ret = -EAGAIN;
378 			break;
379 		}
380 		mutex_unlock(&pipe->mutex);
381 		/*
382 		 * We only get here if we didn't actually read anything.
383 		 *
384 		 * But because we didn't read anything, at this point we can
385 		 * just return directly with -ERESTARTSYS if we're interrupted,
386 		 * since we've done any required wakeups and there's no need
387 		 * to mark anything accessed. And we've dropped the lock.
388 		 */
389 		if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
390 			return -ERESTARTSYS;
391 
392 		wake_next_reader = true;
393 		mutex_lock(&pipe->mutex);
394 	}
395 	if (pipe_is_empty(pipe))
396 		wake_next_reader = false;
397 	mutex_unlock(&pipe->mutex);
398 
399 	if (wake_writer)
400 		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
401 	if (wake_next_reader)
402 		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
403 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
404 	return ret;
405 }
406 
407 static ssize_t
fifo_pipe_read(struct kiocb * iocb,struct iov_iter * to)408 fifo_pipe_read(struct kiocb *iocb, struct iov_iter *to)
409 {
410 	int ret = anon_pipe_read(iocb, to);
411 	if (ret > 0)
412 		file_accessed(iocb->ki_filp);
413 	return ret;
414 }
415 
is_packetized(struct file * file)416 static inline int is_packetized(struct file *file)
417 {
418 	return (file->f_flags & O_DIRECT) != 0;
419 }
420 
421 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
pipe_writable(const struct pipe_inode_info * pipe)422 static inline bool pipe_writable(const struct pipe_inode_info *pipe)
423 {
424 	union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
425 	unsigned int max_usage = READ_ONCE(pipe->max_usage);
426 
427 	return !pipe_full(idx.head, idx.tail, max_usage) ||
428 		!READ_ONCE(pipe->readers);
429 }
430 
431 static ssize_t
anon_pipe_write(struct kiocb * iocb,struct iov_iter * from)432 anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
433 {
434 	struct file *filp = iocb->ki_filp;
435 	struct pipe_inode_info *pipe = filp->private_data;
436 	unsigned int head;
437 	ssize_t ret = 0;
438 	size_t total_len = iov_iter_count(from);
439 	ssize_t chars;
440 	bool was_empty = false;
441 	bool wake_next_writer = false;
442 
443 	/*
444 	 * Reject writing to watch queue pipes before the point where we lock
445 	 * the pipe.
446 	 * Otherwise, lockdep would be unhappy if the caller already has another
447 	 * pipe locked.
448 	 * If we had to support locking a normal pipe and a notification pipe at
449 	 * the same time, we could set up lockdep annotations for that, but
450 	 * since we don't actually need that, it's simpler to just bail here.
451 	 */
452 	if (pipe_has_watch_queue(pipe))
453 		return -EXDEV;
454 
455 	/* Null write succeeds. */
456 	if (unlikely(total_len == 0))
457 		return 0;
458 
459 	mutex_lock(&pipe->mutex);
460 
461 	if (!pipe->readers) {
462 		send_sig(SIGPIPE, current, 0);
463 		ret = -EPIPE;
464 		goto out;
465 	}
466 
467 	/*
468 	 * If it wasn't empty we try to merge new data into
469 	 * the last buffer.
470 	 *
471 	 * That naturally merges small writes, but it also
472 	 * page-aligns the rest of the writes for large writes
473 	 * spanning multiple pages.
474 	 */
475 	head = pipe->head;
476 	was_empty = pipe_empty(head, pipe->tail);
477 	chars = total_len & (PAGE_SIZE-1);
478 	if (chars && !was_empty) {
479 		struct pipe_buffer *buf = pipe_buf(pipe, head - 1);
480 		int offset = buf->offset + buf->len;
481 
482 		if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
483 		    offset + chars <= PAGE_SIZE) {
484 			ret = pipe_buf_confirm(pipe, buf);
485 			if (ret)
486 				goto out;
487 
488 			ret = copy_page_from_iter(buf->page, offset, chars, from);
489 			if (unlikely(ret < chars)) {
490 				ret = -EFAULT;
491 				goto out;
492 			}
493 
494 			buf->len += ret;
495 			if (!iov_iter_count(from))
496 				goto out;
497 		}
498 	}
499 
500 	for (;;) {
501 		if (!pipe->readers) {
502 			send_sig(SIGPIPE, current, 0);
503 			if (!ret)
504 				ret = -EPIPE;
505 			break;
506 		}
507 
508 		head = pipe->head;
509 		if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
510 			struct pipe_buffer *buf;
511 			struct page *page;
512 			int copied;
513 
514 			page = anon_pipe_get_page(pipe);
515 			if (unlikely(!page)) {
516 				if (!ret)
517 					ret = -ENOMEM;
518 				break;
519 			}
520 
521 			copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
522 			if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
523 				anon_pipe_put_page(pipe, page);
524 				if (!ret)
525 					ret = -EFAULT;
526 				break;
527 			}
528 
529 			pipe->head = head + 1;
530 			/* Insert it into the buffer array */
531 			buf = pipe_buf(pipe, head);
532 			buf->page = page;
533 			buf->ops = &anon_pipe_buf_ops;
534 			buf->offset = 0;
535 			if (is_packetized(filp))
536 				buf->flags = PIPE_BUF_FLAG_PACKET;
537 			else
538 				buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
539 
540 			buf->len = copied;
541 			ret += copied;
542 
543 			if (!iov_iter_count(from))
544 				break;
545 
546 			continue;
547 		}
548 
549 		/* Wait for buffer space to become available. */
550 		if ((filp->f_flags & O_NONBLOCK) ||
551 		    (iocb->ki_flags & IOCB_NOWAIT)) {
552 			if (!ret)
553 				ret = -EAGAIN;
554 			break;
555 		}
556 		if (signal_pending(current)) {
557 			if (!ret)
558 				ret = -ERESTARTSYS;
559 			break;
560 		}
561 
562 		/*
563 		 * We're going to release the pipe lock and wait for more
564 		 * space. We wake up any readers if necessary, and then
565 		 * after waiting we need to re-check whether the pipe
566 		 * become empty while we dropped the lock.
567 		 */
568 		mutex_unlock(&pipe->mutex);
569 		if (was_empty)
570 			wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
571 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
572 		wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
573 		mutex_lock(&pipe->mutex);
574 		was_empty = pipe_is_empty(pipe);
575 		wake_next_writer = true;
576 	}
577 out:
578 	if (pipe_is_full(pipe))
579 		wake_next_writer = false;
580 	mutex_unlock(&pipe->mutex);
581 
582 	/*
583 	 * If we do do a wakeup event, we do a 'sync' wakeup, because we
584 	 * want the reader to start processing things asap, rather than
585 	 * leave the data pending.
586 	 *
587 	 * This is particularly important for small writes, because of
588 	 * how (for example) the GNU make jobserver uses small writes to
589 	 * wake up pending jobs
590 	 *
591 	 * Epoll nonsensically wants a wakeup whether the pipe
592 	 * was already empty or not.
593 	 */
594 	if (was_empty || pipe->poll_usage)
595 		wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
596 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
597 	if (wake_next_writer)
598 		wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
599 	return ret;
600 }
601 
602 static ssize_t
fifo_pipe_write(struct kiocb * iocb,struct iov_iter * from)603 fifo_pipe_write(struct kiocb *iocb, struct iov_iter *from)
604 {
605 	int ret = anon_pipe_write(iocb, from);
606 	if (ret > 0) {
607 		struct file *filp = iocb->ki_filp;
608 		if (sb_start_write_trylock(file_inode(filp)->i_sb)) {
609 			int err = file_update_time(filp);
610 			if (err)
611 				ret = err;
612 			sb_end_write(file_inode(filp)->i_sb);
613 		}
614 	}
615 	return ret;
616 }
617 
pipe_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)618 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
619 {
620 	struct pipe_inode_info *pipe = filp->private_data;
621 	unsigned int count, head, tail;
622 
623 	switch (cmd) {
624 	case FIONREAD:
625 		mutex_lock(&pipe->mutex);
626 		count = 0;
627 		head = pipe->head;
628 		tail = pipe->tail;
629 
630 		while (!pipe_empty(head, tail)) {
631 			count += pipe_buf(pipe, tail)->len;
632 			tail++;
633 		}
634 		mutex_unlock(&pipe->mutex);
635 
636 		return put_user(count, (int __user *)arg);
637 
638 #ifdef CONFIG_WATCH_QUEUE
639 	case IOC_WATCH_QUEUE_SET_SIZE: {
640 		int ret;
641 		mutex_lock(&pipe->mutex);
642 		ret = watch_queue_set_size(pipe, arg);
643 		mutex_unlock(&pipe->mutex);
644 		return ret;
645 	}
646 
647 	case IOC_WATCH_QUEUE_SET_FILTER:
648 		return watch_queue_set_filter(
649 			pipe, (struct watch_notification_filter __user *)arg);
650 #endif
651 
652 	default:
653 		return -ENOIOCTLCMD;
654 	}
655 }
656 
657 /* No kernel lock held - fine */
658 static __poll_t
pipe_poll(struct file * filp,poll_table * wait)659 pipe_poll(struct file *filp, poll_table *wait)
660 {
661 	__poll_t mask;
662 	struct pipe_inode_info *pipe = filp->private_data;
663 	union pipe_index idx;
664 
665 	/* Epoll has some historical nasty semantics, this enables them */
666 	WRITE_ONCE(pipe->poll_usage, true);
667 
668 	/*
669 	 * Reading pipe state only -- no need for acquiring the semaphore.
670 	 *
671 	 * But because this is racy, the code has to add the
672 	 * entry to the poll table _first_ ..
673 	 */
674 	if (filp->f_mode & FMODE_READ)
675 		poll_wait(filp, &pipe->rd_wait, wait);
676 	if (filp->f_mode & FMODE_WRITE)
677 		poll_wait(filp, &pipe->wr_wait, wait);
678 
679 	/*
680 	 * .. and only then can you do the racy tests. That way,
681 	 * if something changes and you got it wrong, the poll
682 	 * table entry will wake you up and fix it.
683 	 */
684 	idx.head_tail = READ_ONCE(pipe->head_tail);
685 
686 	mask = 0;
687 	if (filp->f_mode & FMODE_READ) {
688 		if (!pipe_empty(idx.head, idx.tail))
689 			mask |= EPOLLIN | EPOLLRDNORM;
690 		if (!pipe->writers && filp->f_pipe != pipe->w_counter)
691 			mask |= EPOLLHUP;
692 	}
693 
694 	if (filp->f_mode & FMODE_WRITE) {
695 		if (!pipe_full(idx.head, idx.tail, pipe->max_usage))
696 			mask |= EPOLLOUT | EPOLLWRNORM;
697 		/*
698 		 * Most Unices do not set EPOLLERR for FIFOs but on Linux they
699 		 * behave exactly like pipes for poll().
700 		 */
701 		if (!pipe->readers)
702 			mask |= EPOLLERR;
703 	}
704 
705 	return mask;
706 }
707 
put_pipe_info(struct inode * inode,struct pipe_inode_info * pipe)708 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
709 {
710 	int kill = 0;
711 
712 	spin_lock(&inode->i_lock);
713 	if (!--pipe->files) {
714 		inode->i_pipe = NULL;
715 		kill = 1;
716 	}
717 	spin_unlock(&inode->i_lock);
718 
719 	if (kill)
720 		free_pipe_info(pipe);
721 }
722 
723 static int
pipe_release(struct inode * inode,struct file * file)724 pipe_release(struct inode *inode, struct file *file)
725 {
726 	struct pipe_inode_info *pipe = file->private_data;
727 
728 	mutex_lock(&pipe->mutex);
729 	if (file->f_mode & FMODE_READ)
730 		pipe->readers--;
731 	if (file->f_mode & FMODE_WRITE)
732 		pipe->writers--;
733 
734 	/* Was that the last reader or writer, but not the other side? */
735 	if (!pipe->readers != !pipe->writers) {
736 		wake_up_interruptible_all(&pipe->rd_wait);
737 		wake_up_interruptible_all(&pipe->wr_wait);
738 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
739 		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
740 	}
741 	mutex_unlock(&pipe->mutex);
742 
743 	put_pipe_info(inode, pipe);
744 	return 0;
745 }
746 
747 static int
pipe_fasync(int fd,struct file * filp,int on)748 pipe_fasync(int fd, struct file *filp, int on)
749 {
750 	struct pipe_inode_info *pipe = filp->private_data;
751 	int retval = 0;
752 
753 	mutex_lock(&pipe->mutex);
754 	if (filp->f_mode & FMODE_READ)
755 		retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
756 	if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
757 		retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
758 		if (retval < 0 && (filp->f_mode & FMODE_READ))
759 			/* this can happen only if on == T */
760 			fasync_helper(-1, filp, 0, &pipe->fasync_readers);
761 	}
762 	mutex_unlock(&pipe->mutex);
763 	return retval;
764 }
765 
account_pipe_buffers(struct user_struct * user,unsigned long old,unsigned long new)766 unsigned long account_pipe_buffers(struct user_struct *user,
767 				   unsigned long old, unsigned long new)
768 {
769 	return atomic_long_add_return(new - old, &user->pipe_bufs);
770 }
771 
too_many_pipe_buffers_soft(unsigned long user_bufs)772 bool too_many_pipe_buffers_soft(unsigned long user_bufs)
773 {
774 	unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
775 
776 	return soft_limit && user_bufs > soft_limit;
777 }
778 
too_many_pipe_buffers_hard(unsigned long user_bufs)779 bool too_many_pipe_buffers_hard(unsigned long user_bufs)
780 {
781 	unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
782 
783 	return hard_limit && user_bufs > hard_limit;
784 }
785 
pipe_is_unprivileged_user(void)786 bool pipe_is_unprivileged_user(void)
787 {
788 	return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
789 }
790 
alloc_pipe_info(void)791 struct pipe_inode_info *alloc_pipe_info(void)
792 {
793 	struct pipe_inode_info *pipe;
794 	unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
795 	struct user_struct *user = get_current_user();
796 	unsigned long user_bufs;
797 	unsigned int max_size = READ_ONCE(pipe_max_size);
798 
799 	pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
800 	if (pipe == NULL)
801 		goto out_free_uid;
802 
803 	if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
804 		pipe_bufs = max_size >> PAGE_SHIFT;
805 
806 	user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
807 
808 	if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
809 		user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
810 		pipe_bufs = PIPE_MIN_DEF_BUFFERS;
811 	}
812 
813 	if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
814 		goto out_revert_acct;
815 
816 	pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
817 			     GFP_KERNEL_ACCOUNT);
818 
819 	if (pipe->bufs) {
820 		init_waitqueue_head(&pipe->rd_wait);
821 		init_waitqueue_head(&pipe->wr_wait);
822 		pipe->r_counter = pipe->w_counter = 1;
823 		pipe->max_usage = pipe_bufs;
824 		pipe->ring_size = pipe_bufs;
825 		pipe->nr_accounted = pipe_bufs;
826 		pipe->user = user;
827 		mutex_init(&pipe->mutex);
828 		lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL);
829 		return pipe;
830 	}
831 
832 out_revert_acct:
833 	(void) account_pipe_buffers(user, pipe_bufs, 0);
834 	kfree(pipe);
835 out_free_uid:
836 	free_uid(user);
837 	return NULL;
838 }
839 
free_pipe_info(struct pipe_inode_info * pipe)840 void free_pipe_info(struct pipe_inode_info *pipe)
841 {
842 	unsigned int i;
843 
844 #ifdef CONFIG_WATCH_QUEUE
845 	if (pipe->watch_queue)
846 		watch_queue_clear(pipe->watch_queue);
847 #endif
848 
849 	(void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
850 	free_uid(pipe->user);
851 	for (i = 0; i < pipe->ring_size; i++) {
852 		struct pipe_buffer *buf = pipe->bufs + i;
853 		if (buf->ops)
854 			pipe_buf_release(pipe, buf);
855 	}
856 #ifdef CONFIG_WATCH_QUEUE
857 	if (pipe->watch_queue)
858 		put_watch_queue(pipe->watch_queue);
859 #endif
860 	for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
861 		if (pipe->tmp_page[i])
862 			__free_page(pipe->tmp_page[i]);
863 	}
864 	kfree(pipe->bufs);
865 	kfree(pipe);
866 }
867 
868 static struct vfsmount *pipe_mnt __ro_after_init;
869 
870 /*
871  * pipefs_dname() is called from d_path().
872  */
pipefs_dname(struct dentry * dentry,char * buffer,int buflen)873 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
874 {
875 	return dynamic_dname(buffer, buflen, "pipe:[%lu]",
876 				d_inode(dentry)->i_ino);
877 }
878 
879 static const struct dentry_operations pipefs_dentry_operations = {
880 	.d_dname	= pipefs_dname,
881 };
882 
883 static const struct file_operations pipeanon_fops;
884 
get_pipe_inode(void)885 static struct inode * get_pipe_inode(void)
886 {
887 	struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
888 	struct pipe_inode_info *pipe;
889 
890 	if (!inode)
891 		goto fail_inode;
892 
893 	inode->i_ino = get_next_ino();
894 
895 	pipe = alloc_pipe_info();
896 	if (!pipe)
897 		goto fail_iput;
898 
899 	inode->i_pipe = pipe;
900 	pipe->files = 2;
901 	pipe->readers = pipe->writers = 1;
902 	inode->i_fop = &pipeanon_fops;
903 
904 	/*
905 	 * Mark the inode dirty from the very beginning,
906 	 * that way it will never be moved to the dirty
907 	 * list because "mark_inode_dirty()" will think
908 	 * that it already _is_ on the dirty list.
909 	 */
910 	inode->i_state = I_DIRTY;
911 	inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
912 	inode->i_uid = current_fsuid();
913 	inode->i_gid = current_fsgid();
914 	simple_inode_init_ts(inode);
915 
916 	return inode;
917 
918 fail_iput:
919 	iput(inode);
920 
921 fail_inode:
922 	return NULL;
923 }
924 
create_pipe_files(struct file ** res,int flags)925 int create_pipe_files(struct file **res, int flags)
926 {
927 	struct inode *inode = get_pipe_inode();
928 	struct file *f;
929 	int error;
930 
931 	if (!inode)
932 		return -ENFILE;
933 
934 	if (flags & O_NOTIFICATION_PIPE) {
935 		error = watch_queue_init(inode->i_pipe);
936 		if (error) {
937 			free_pipe_info(inode->i_pipe);
938 			iput(inode);
939 			return error;
940 		}
941 	}
942 
943 	f = alloc_file_pseudo(inode, pipe_mnt, "",
944 				O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
945 				&pipeanon_fops);
946 	if (IS_ERR(f)) {
947 		free_pipe_info(inode->i_pipe);
948 		iput(inode);
949 		return PTR_ERR(f);
950 	}
951 
952 	f->private_data = inode->i_pipe;
953 	f->f_pipe = 0;
954 
955 	res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),
956 				  &pipeanon_fops);
957 	if (IS_ERR(res[0])) {
958 		put_pipe_info(inode, inode->i_pipe);
959 		fput(f);
960 		return PTR_ERR(res[0]);
961 	}
962 	res[0]->private_data = inode->i_pipe;
963 	res[0]->f_pipe = 0;
964 	res[1] = f;
965 	stream_open(inode, res[0]);
966 	stream_open(inode, res[1]);
967 	/*
968 	 * Disable permission and pre-content events, but enable legacy
969 	 * inotify events for legacy users.
970 	 */
971 	file_set_fsnotify_mode(res[0], FMODE_NONOTIFY_PERM);
972 	file_set_fsnotify_mode(res[1], FMODE_NONOTIFY_PERM);
973 	return 0;
974 }
975 
__do_pipe_flags(int * fd,struct file ** files,int flags)976 static int __do_pipe_flags(int *fd, struct file **files, int flags)
977 {
978 	int error;
979 	int fdw, fdr;
980 
981 	if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
982 		return -EINVAL;
983 
984 	error = create_pipe_files(files, flags);
985 	if (error)
986 		return error;
987 
988 	error = get_unused_fd_flags(flags);
989 	if (error < 0)
990 		goto err_read_pipe;
991 	fdr = error;
992 
993 	error = get_unused_fd_flags(flags);
994 	if (error < 0)
995 		goto err_fdr;
996 	fdw = error;
997 
998 	audit_fd_pair(fdr, fdw);
999 	fd[0] = fdr;
1000 	fd[1] = fdw;
1001 	/* pipe groks IOCB_NOWAIT */
1002 	files[0]->f_mode |= FMODE_NOWAIT;
1003 	files[1]->f_mode |= FMODE_NOWAIT;
1004 	return 0;
1005 
1006  err_fdr:
1007 	put_unused_fd(fdr);
1008  err_read_pipe:
1009 	fput(files[0]);
1010 	fput(files[1]);
1011 	return error;
1012 }
1013 
do_pipe_flags(int * fd,int flags)1014 int do_pipe_flags(int *fd, int flags)
1015 {
1016 	struct file *files[2];
1017 	int error = __do_pipe_flags(fd, files, flags);
1018 	if (!error) {
1019 		fd_install(fd[0], files[0]);
1020 		fd_install(fd[1], files[1]);
1021 	}
1022 	return error;
1023 }
1024 
1025 /*
1026  * sys_pipe() is the normal C calling standard for creating
1027  * a pipe. It's not the way Unix traditionally does this, though.
1028  */
do_pipe2(int __user * fildes,int flags)1029 static int do_pipe2(int __user *fildes, int flags)
1030 {
1031 	struct file *files[2];
1032 	int fd[2];
1033 	int error;
1034 
1035 	error = __do_pipe_flags(fd, files, flags);
1036 	if (!error) {
1037 		if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
1038 			fput(files[0]);
1039 			fput(files[1]);
1040 			put_unused_fd(fd[0]);
1041 			put_unused_fd(fd[1]);
1042 			error = -EFAULT;
1043 		} else {
1044 			fd_install(fd[0], files[0]);
1045 			fd_install(fd[1], files[1]);
1046 		}
1047 	}
1048 	return error;
1049 }
1050 
SYSCALL_DEFINE2(pipe2,int __user *,fildes,int,flags)1051 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
1052 {
1053 	return do_pipe2(fildes, flags);
1054 }
1055 
SYSCALL_DEFINE1(pipe,int __user *,fildes)1056 SYSCALL_DEFINE1(pipe, int __user *, fildes)
1057 {
1058 	return do_pipe2(fildes, 0);
1059 }
1060 
1061 /*
1062  * This is the stupid "wait for pipe to be readable or writable"
1063  * model.
1064  *
1065  * See pipe_read/write() for the proper kind of exclusive wait,
1066  * but that requires that we wake up any other readers/writers
1067  * if we then do not end up reading everything (ie the whole
1068  * "wake_next_reader/writer" logic in pipe_read/write()).
1069  */
pipe_wait_readable(struct pipe_inode_info * pipe)1070 void pipe_wait_readable(struct pipe_inode_info *pipe)
1071 {
1072 	pipe_unlock(pipe);
1073 	wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
1074 	pipe_lock(pipe);
1075 }
1076 
pipe_wait_writable(struct pipe_inode_info * pipe)1077 void pipe_wait_writable(struct pipe_inode_info *pipe)
1078 {
1079 	pipe_unlock(pipe);
1080 	wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
1081 	pipe_lock(pipe);
1082 }
1083 
1084 /*
1085  * This depends on both the wait (here) and the wakeup (wake_up_partner)
1086  * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1087  * race with the count check and waitqueue prep.
1088  *
1089  * Normally in order to avoid races, you'd do the prepare_to_wait() first,
1090  * then check the condition you're waiting for, and only then sleep. But
1091  * because of the pipe lock, we can check the condition before being on
1092  * the wait queue.
1093  *
1094  * We use the 'rd_wait' waitqueue for pipe partner waiting.
1095  */
wait_for_partner(struct pipe_inode_info * pipe,unsigned int * cnt)1096 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
1097 {
1098 	DEFINE_WAIT(rdwait);
1099 	int cur = *cnt;
1100 
1101 	while (cur == *cnt) {
1102 		prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
1103 		pipe_unlock(pipe);
1104 		schedule();
1105 		finish_wait(&pipe->rd_wait, &rdwait);
1106 		pipe_lock(pipe);
1107 		if (signal_pending(current))
1108 			break;
1109 	}
1110 	return cur == *cnt ? -ERESTARTSYS : 0;
1111 }
1112 
wake_up_partner(struct pipe_inode_info * pipe)1113 static void wake_up_partner(struct pipe_inode_info *pipe)
1114 {
1115 	wake_up_interruptible_all(&pipe->rd_wait);
1116 }
1117 
fifo_open(struct inode * inode,struct file * filp)1118 static int fifo_open(struct inode *inode, struct file *filp)
1119 {
1120 	bool is_pipe = inode->i_fop == &pipeanon_fops;
1121 	struct pipe_inode_info *pipe;
1122 	int ret;
1123 
1124 	filp->f_pipe = 0;
1125 
1126 	spin_lock(&inode->i_lock);
1127 	if (inode->i_pipe) {
1128 		pipe = inode->i_pipe;
1129 		pipe->files++;
1130 		spin_unlock(&inode->i_lock);
1131 	} else {
1132 		spin_unlock(&inode->i_lock);
1133 		pipe = alloc_pipe_info();
1134 		if (!pipe)
1135 			return -ENOMEM;
1136 		pipe->files = 1;
1137 		spin_lock(&inode->i_lock);
1138 		if (unlikely(inode->i_pipe)) {
1139 			inode->i_pipe->files++;
1140 			spin_unlock(&inode->i_lock);
1141 			free_pipe_info(pipe);
1142 			pipe = inode->i_pipe;
1143 		} else {
1144 			inode->i_pipe = pipe;
1145 			spin_unlock(&inode->i_lock);
1146 		}
1147 	}
1148 	filp->private_data = pipe;
1149 	/* OK, we have a pipe and it's pinned down */
1150 
1151 	mutex_lock(&pipe->mutex);
1152 
1153 	/* We can only do regular read/write on fifos */
1154 	stream_open(inode, filp);
1155 
1156 	switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {
1157 	case FMODE_READ:
1158 	/*
1159 	 *  O_RDONLY
1160 	 *  POSIX.1 says that O_NONBLOCK means return with the FIFO
1161 	 *  opened, even when there is no process writing the FIFO.
1162 	 */
1163 		pipe->r_counter++;
1164 		if (pipe->readers++ == 0)
1165 			wake_up_partner(pipe);
1166 
1167 		if (!is_pipe && !pipe->writers) {
1168 			if ((filp->f_flags & O_NONBLOCK)) {
1169 				/* suppress EPOLLHUP until we have
1170 				 * seen a writer */
1171 				filp->f_pipe = pipe->w_counter;
1172 			} else {
1173 				if (wait_for_partner(pipe, &pipe->w_counter))
1174 					goto err_rd;
1175 			}
1176 		}
1177 		break;
1178 
1179 	case FMODE_WRITE:
1180 	/*
1181 	 *  O_WRONLY
1182 	 *  POSIX.1 says that O_NONBLOCK means return -1 with
1183 	 *  errno=ENXIO when there is no process reading the FIFO.
1184 	 */
1185 		ret = -ENXIO;
1186 		if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
1187 			goto err;
1188 
1189 		pipe->w_counter++;
1190 		if (!pipe->writers++)
1191 			wake_up_partner(pipe);
1192 
1193 		if (!is_pipe && !pipe->readers) {
1194 			if (wait_for_partner(pipe, &pipe->r_counter))
1195 				goto err_wr;
1196 		}
1197 		break;
1198 
1199 	case FMODE_READ | FMODE_WRITE:
1200 	/*
1201 	 *  O_RDWR
1202 	 *  POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
1203 	 *  This implementation will NEVER block on a O_RDWR open, since
1204 	 *  the process can at least talk to itself.
1205 	 */
1206 
1207 		pipe->readers++;
1208 		pipe->writers++;
1209 		pipe->r_counter++;
1210 		pipe->w_counter++;
1211 		if (pipe->readers == 1 || pipe->writers == 1)
1212 			wake_up_partner(pipe);
1213 		break;
1214 
1215 	default:
1216 		ret = -EINVAL;
1217 		goto err;
1218 	}
1219 
1220 	/* Ok! */
1221 	mutex_unlock(&pipe->mutex);
1222 	return 0;
1223 
1224 err_rd:
1225 	if (!--pipe->readers)
1226 		wake_up_interruptible(&pipe->wr_wait);
1227 	ret = -ERESTARTSYS;
1228 	goto err;
1229 
1230 err_wr:
1231 	if (!--pipe->writers)
1232 		wake_up_interruptible_all(&pipe->rd_wait);
1233 	ret = -ERESTARTSYS;
1234 	goto err;
1235 
1236 err:
1237 	mutex_unlock(&pipe->mutex);
1238 
1239 	put_pipe_info(inode, pipe);
1240 	return ret;
1241 }
1242 
1243 const struct file_operations pipefifo_fops = {
1244 	.open		= fifo_open,
1245 	.read_iter	= fifo_pipe_read,
1246 	.write_iter	= fifo_pipe_write,
1247 	.poll		= pipe_poll,
1248 	.unlocked_ioctl	= pipe_ioctl,
1249 	.release	= pipe_release,
1250 	.fasync		= pipe_fasync,
1251 	.splice_write	= iter_file_splice_write,
1252 };
1253 
1254 static const struct file_operations pipeanon_fops = {
1255 	.open		= fifo_open,
1256 	.read_iter	= anon_pipe_read,
1257 	.write_iter	= anon_pipe_write,
1258 	.poll		= pipe_poll,
1259 	.unlocked_ioctl	= pipe_ioctl,
1260 	.release	= pipe_release,
1261 	.fasync		= pipe_fasync,
1262 	.splice_write	= iter_file_splice_write,
1263 };
1264 
1265 /*
1266  * Currently we rely on the pipe array holding a power-of-2 number
1267  * of pages. Returns 0 on error.
1268  */
round_pipe_size(unsigned int size)1269 unsigned int round_pipe_size(unsigned int size)
1270 {
1271 	if (size > (1U << 31))
1272 		return 0;
1273 
1274 	/* Minimum pipe size, as required by POSIX */
1275 	if (size < PAGE_SIZE)
1276 		return PAGE_SIZE;
1277 
1278 	return roundup_pow_of_two(size);
1279 }
1280 
1281 /*
1282  * Resize the pipe ring to a number of slots.
1283  *
1284  * Note the pipe can be reduced in capacity, but only if the current
1285  * occupancy doesn't exceed nr_slots; if it does, EBUSY will be
1286  * returned instead.
1287  */
pipe_resize_ring(struct pipe_inode_info * pipe,unsigned int nr_slots)1288 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
1289 {
1290 	struct pipe_buffer *bufs;
1291 	unsigned int head, tail, mask, n;
1292 
1293 	/* nr_slots larger than limits of pipe->{head,tail} */
1294 	if (unlikely(nr_slots > (pipe_index_t)-1u))
1295 		return -EINVAL;
1296 
1297 	bufs = kcalloc(nr_slots, sizeof(*bufs),
1298 		       GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
1299 	if (unlikely(!bufs))
1300 		return -ENOMEM;
1301 
1302 	spin_lock_irq(&pipe->rd_wait.lock);
1303 	mask = pipe->ring_size - 1;
1304 	head = pipe->head;
1305 	tail = pipe->tail;
1306 
1307 	n = pipe_occupancy(head, tail);
1308 	if (nr_slots < n) {
1309 		spin_unlock_irq(&pipe->rd_wait.lock);
1310 		kfree(bufs);
1311 		return -EBUSY;
1312 	}
1313 
1314 	/*
1315 	 * The pipe array wraps around, so just start the new one at zero
1316 	 * and adjust the indices.
1317 	 */
1318 	if (n > 0) {
1319 		unsigned int h = head & mask;
1320 		unsigned int t = tail & mask;
1321 		if (h > t) {
1322 			memcpy(bufs, pipe->bufs + t,
1323 			       n * sizeof(struct pipe_buffer));
1324 		} else {
1325 			unsigned int tsize = pipe->ring_size - t;
1326 			if (h > 0)
1327 				memcpy(bufs + tsize, pipe->bufs,
1328 				       h * sizeof(struct pipe_buffer));
1329 			memcpy(bufs, pipe->bufs + t,
1330 			       tsize * sizeof(struct pipe_buffer));
1331 		}
1332 	}
1333 
1334 	head = n;
1335 	tail = 0;
1336 
1337 	kfree(pipe->bufs);
1338 	pipe->bufs = bufs;
1339 	pipe->ring_size = nr_slots;
1340 	if (pipe->max_usage > nr_slots)
1341 		pipe->max_usage = nr_slots;
1342 	pipe->tail = tail;
1343 	pipe->head = head;
1344 
1345 	if (!pipe_has_watch_queue(pipe)) {
1346 		pipe->max_usage = nr_slots;
1347 		pipe->nr_accounted = nr_slots;
1348 	}
1349 
1350 	spin_unlock_irq(&pipe->rd_wait.lock);
1351 
1352 	/* This might have made more room for writers */
1353 	wake_up_interruptible(&pipe->wr_wait);
1354 	return 0;
1355 }
1356 
1357 /*
1358  * Allocate a new array of pipe buffers and copy the info over. Returns the
1359  * pipe size if successful, or return -ERROR on error.
1360  */
pipe_set_size(struct pipe_inode_info * pipe,unsigned int arg)1361 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg)
1362 {
1363 	unsigned long user_bufs;
1364 	unsigned int nr_slots, size;
1365 	long ret = 0;
1366 
1367 	if (pipe_has_watch_queue(pipe))
1368 		return -EBUSY;
1369 
1370 	size = round_pipe_size(arg);
1371 	nr_slots = size >> PAGE_SHIFT;
1372 
1373 	if (!nr_slots)
1374 		return -EINVAL;
1375 
1376 	/*
1377 	 * If trying to increase the pipe capacity, check that an
1378 	 * unprivileged user is not trying to exceed various limits
1379 	 * (soft limit check here, hard limit check just below).
1380 	 * Decreasing the pipe capacity is always permitted, even
1381 	 * if the user is currently over a limit.
1382 	 */
1383 	if (nr_slots > pipe->max_usage &&
1384 			size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
1385 		return -EPERM;
1386 
1387 	user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
1388 
1389 	if (nr_slots > pipe->max_usage &&
1390 			(too_many_pipe_buffers_hard(user_bufs) ||
1391 			 too_many_pipe_buffers_soft(user_bufs)) &&
1392 			pipe_is_unprivileged_user()) {
1393 		ret = -EPERM;
1394 		goto out_revert_acct;
1395 	}
1396 
1397 	ret = pipe_resize_ring(pipe, nr_slots);
1398 	if (ret < 0)
1399 		goto out_revert_acct;
1400 
1401 	return pipe->max_usage * PAGE_SIZE;
1402 
1403 out_revert_acct:
1404 	(void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
1405 	return ret;
1406 }
1407 
1408 /*
1409  * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is
1410  * not enough to verify that this is a pipe.
1411  */
get_pipe_info(struct file * file,bool for_splice)1412 struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
1413 {
1414 	struct pipe_inode_info *pipe = file->private_data;
1415 
1416 	if (!pipe)
1417 		return NULL;
1418 	if (file->f_op != &pipefifo_fops && file->f_op != &pipeanon_fops)
1419 		return NULL;
1420 	if (for_splice && pipe_has_watch_queue(pipe))
1421 		return NULL;
1422 	return pipe;
1423 }
1424 
pipe_fcntl(struct file * file,unsigned int cmd,unsigned int arg)1425 long pipe_fcntl(struct file *file, unsigned int cmd, unsigned int arg)
1426 {
1427 	struct pipe_inode_info *pipe;
1428 	long ret;
1429 
1430 	pipe = get_pipe_info(file, false);
1431 	if (!pipe)
1432 		return -EBADF;
1433 
1434 	mutex_lock(&pipe->mutex);
1435 
1436 	switch (cmd) {
1437 	case F_SETPIPE_SZ:
1438 		ret = pipe_set_size(pipe, arg);
1439 		break;
1440 	case F_GETPIPE_SZ:
1441 		ret = pipe->max_usage * PAGE_SIZE;
1442 		break;
1443 	default:
1444 		ret = -EINVAL;
1445 		break;
1446 	}
1447 
1448 	mutex_unlock(&pipe->mutex);
1449 	return ret;
1450 }
1451 
1452 static const struct super_operations pipefs_ops = {
1453 	.destroy_inode = free_inode_nonrcu,
1454 	.statfs = simple_statfs,
1455 };
1456 
1457 /*
1458  * pipefs should _never_ be mounted by userland - too much of security hassle,
1459  * no real gain from having the whole file system mounted. So we don't need
1460  * any operations on the root directory. However, we need a non-trivial
1461  * d_name - pipe: will go nicely and kill the special-casing in procfs.
1462  */
1463 
pipefs_init_fs_context(struct fs_context * fc)1464 static int pipefs_init_fs_context(struct fs_context *fc)
1465 {
1466 	struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
1467 	if (!ctx)
1468 		return -ENOMEM;
1469 	ctx->ops = &pipefs_ops;
1470 	ctx->dops = &pipefs_dentry_operations;
1471 	return 0;
1472 }
1473 
1474 static struct file_system_type pipe_fs_type = {
1475 	.name		= "pipefs",
1476 	.init_fs_context = pipefs_init_fs_context,
1477 	.kill_sb	= kill_anon_super,
1478 };
1479 
1480 #ifdef CONFIG_SYSCTL
do_proc_dopipe_max_size_conv(unsigned long * lvalp,unsigned int * valp,int write,void * data)1481 static int do_proc_dopipe_max_size_conv(unsigned long *lvalp,
1482 					unsigned int *valp,
1483 					int write, void *data)
1484 {
1485 	if (write) {
1486 		unsigned int val;
1487 
1488 		val = round_pipe_size(*lvalp);
1489 		if (val == 0)
1490 			return -EINVAL;
1491 
1492 		*valp = val;
1493 	} else {
1494 		unsigned int val = *valp;
1495 		*lvalp = (unsigned long) val;
1496 	}
1497 
1498 	return 0;
1499 }
1500 
proc_dopipe_max_size(const struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)1501 static int proc_dopipe_max_size(const struct ctl_table *table, int write,
1502 				void *buffer, size_t *lenp, loff_t *ppos)
1503 {
1504 	return do_proc_douintvec(table, write, buffer, lenp, ppos,
1505 				 do_proc_dopipe_max_size_conv, NULL);
1506 }
1507 
1508 static const struct ctl_table fs_pipe_sysctls[] = {
1509 	{
1510 		.procname	= "pipe-max-size",
1511 		.data		= &pipe_max_size,
1512 		.maxlen		= sizeof(pipe_max_size),
1513 		.mode		= 0644,
1514 		.proc_handler	= proc_dopipe_max_size,
1515 	},
1516 	{
1517 		.procname	= "pipe-user-pages-hard",
1518 		.data		= &pipe_user_pages_hard,
1519 		.maxlen		= sizeof(pipe_user_pages_hard),
1520 		.mode		= 0644,
1521 		.proc_handler	= proc_doulongvec_minmax,
1522 	},
1523 	{
1524 		.procname	= "pipe-user-pages-soft",
1525 		.data		= &pipe_user_pages_soft,
1526 		.maxlen		= sizeof(pipe_user_pages_soft),
1527 		.mode		= 0644,
1528 		.proc_handler	= proc_doulongvec_minmax,
1529 	},
1530 };
1531 #endif
1532 
init_pipe_fs(void)1533 static int __init init_pipe_fs(void)
1534 {
1535 	int err = register_filesystem(&pipe_fs_type);
1536 
1537 	if (!err) {
1538 		pipe_mnt = kern_mount(&pipe_fs_type);
1539 		if (IS_ERR(pipe_mnt)) {
1540 			err = PTR_ERR(pipe_mnt);
1541 			unregister_filesystem(&pipe_fs_type);
1542 		}
1543 	}
1544 #ifdef CONFIG_SYSCTL
1545 	register_sysctl_init("fs", fs_pipe_sysctls);
1546 #endif
1547 	return err;
1548 }
1549 
1550 fs_initcall(init_pipe_fs);
1551