1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * linux/fs/pipe.c
4 *
5 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
6 */
7
8 #include <linux/mm.h>
9 #include <linux/file.h>
10 #include <linux/poll.h>
11 #include <linux/slab.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/fs.h>
15 #include <linux/log2.h>
16 #include <linux/mount.h>
17 #include <linux/pseudo_fs.h>
18 #include <linux/magic.h>
19 #include <linux/pipe_fs_i.h>
20 #include <linux/uio.h>
21 #include <linux/highmem.h>
22 #include <linux/pagemap.h>
23 #include <linux/audit.h>
24 #include <linux/syscalls.h>
25 #include <linux/fcntl.h>
26 #include <linux/memcontrol.h>
27 #include <linux/watch_queue.h>
28 #include <linux/sysctl.h>
29
30 #include <linux/uaccess.h>
31 #include <asm/ioctls.h>
32
33 #include "internal.h"
34
35 /*
36 * New pipe buffers will be restricted to this size while the user is exceeding
37 * their pipe buffer quota. The general pipe use case needs at least two
38 * buffers: one for data yet to be read, and one for new data. If this is less
39 * than two, then a write to a non-empty pipe may block even if the pipe is not
40 * full. This can occur with GNU make jobserver or similar uses of pipes as
41 * semaphores: multiple processes may be waiting to write tokens back to the
42 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
43 *
44 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
45 * own risk, namely: pipe writes to non-full pipes may block until the pipe is
46 * emptied.
47 */
48 #define PIPE_MIN_DEF_BUFFERS 2
49
50 /*
51 * The max size that a non-root user is allowed to grow the pipe. Can
52 * be set by root in /proc/sys/fs/pipe-max-size
53 */
54 static unsigned int pipe_max_size = 1048576;
55
56 /* Maximum allocatable pages per user. Hard limit is unset by default, soft
57 * matches default values.
58 */
59 static unsigned long pipe_user_pages_hard;
60 static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
61
62 /*
63 * We use head and tail indices that aren't masked off, except at the point of
64 * dereference, but rather they're allowed to wrap naturally. This means there
65 * isn't a dead spot in the buffer, but the ring has to be a power of two and
66 * <= 2^31.
67 * -- David Howells 2019-09-23.
68 *
69 * Reads with count = 0 should always return 0.
70 * -- Julian Bradfield 1999-06-07.
71 *
72 * FIFOs and Pipes now generate SIGIO for both readers and writers.
73 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
74 *
75 * pipe_read & write cleanup
76 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
77 */
78
79 #define cmp_int(l, r) ((l > r) - (l < r))
80
81 #ifdef CONFIG_PROVE_LOCKING
pipe_lock_cmp_fn(const struct lockdep_map * a,const struct lockdep_map * b)82 static int pipe_lock_cmp_fn(const struct lockdep_map *a,
83 const struct lockdep_map *b)
84 {
85 return cmp_int((unsigned long) a, (unsigned long) b);
86 }
87 #endif
88
pipe_lock(struct pipe_inode_info * pipe)89 void pipe_lock(struct pipe_inode_info *pipe)
90 {
91 if (pipe->files)
92 mutex_lock(&pipe->mutex);
93 }
94 EXPORT_SYMBOL(pipe_lock);
95
pipe_unlock(struct pipe_inode_info * pipe)96 void pipe_unlock(struct pipe_inode_info *pipe)
97 {
98 if (pipe->files)
99 mutex_unlock(&pipe->mutex);
100 }
101 EXPORT_SYMBOL(pipe_unlock);
102
pipe_double_lock(struct pipe_inode_info * pipe1,struct pipe_inode_info * pipe2)103 void pipe_double_lock(struct pipe_inode_info *pipe1,
104 struct pipe_inode_info *pipe2)
105 {
106 BUG_ON(pipe1 == pipe2);
107
108 if (pipe1 > pipe2)
109 swap(pipe1, pipe2);
110
111 pipe_lock(pipe1);
112 pipe_lock(pipe2);
113 }
114
anon_pipe_get_page(struct pipe_inode_info * pipe)115 static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe)
116 {
117 for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
118 if (pipe->tmp_page[i]) {
119 struct page *page = pipe->tmp_page[i];
120 pipe->tmp_page[i] = NULL;
121 return page;
122 }
123 }
124
125 return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
126 }
127
anon_pipe_put_page(struct pipe_inode_info * pipe,struct page * page)128 static void anon_pipe_put_page(struct pipe_inode_info *pipe,
129 struct page *page)
130 {
131 if (page_count(page) == 1) {
132 for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
133 if (!pipe->tmp_page[i]) {
134 pipe->tmp_page[i] = page;
135 return;
136 }
137 }
138 }
139
140 put_page(page);
141 }
142
anon_pipe_buf_release(struct pipe_inode_info * pipe,struct pipe_buffer * buf)143 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
144 struct pipe_buffer *buf)
145 {
146 struct page *page = buf->page;
147
148 anon_pipe_put_page(pipe, page);
149 }
150
anon_pipe_buf_try_steal(struct pipe_inode_info * pipe,struct pipe_buffer * buf)151 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
152 struct pipe_buffer *buf)
153 {
154 struct page *page = buf->page;
155
156 if (page_count(page) != 1)
157 return false;
158 memcg_kmem_uncharge_page(page, 0);
159 __SetPageLocked(page);
160 return true;
161 }
162
163 /**
164 * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
165 * @pipe: the pipe that the buffer belongs to
166 * @buf: the buffer to attempt to steal
167 *
168 * Description:
169 * This function attempts to steal the &struct page attached to
170 * @buf. If successful, this function returns 0 and returns with
171 * the page locked. The caller may then reuse the page for whatever
172 * he wishes; the typical use is insertion into a different file
173 * page cache.
174 */
generic_pipe_buf_try_steal(struct pipe_inode_info * pipe,struct pipe_buffer * buf)175 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
176 struct pipe_buffer *buf)
177 {
178 struct page *page = buf->page;
179
180 /*
181 * A reference of one is golden, that means that the owner of this
182 * page is the only one holding a reference to it. lock the page
183 * and return OK.
184 */
185 if (page_count(page) == 1) {
186 lock_page(page);
187 return true;
188 }
189 return false;
190 }
191 EXPORT_SYMBOL(generic_pipe_buf_try_steal);
192
193 /**
194 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
195 * @pipe: the pipe that the buffer belongs to
196 * @buf: the buffer to get a reference to
197 *
198 * Description:
199 * This function grabs an extra reference to @buf. It's used in
200 * the tee() system call, when we duplicate the buffers in one
201 * pipe into another.
202 */
generic_pipe_buf_get(struct pipe_inode_info * pipe,struct pipe_buffer * buf)203 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
204 {
205 return try_get_page(buf->page);
206 }
207 EXPORT_SYMBOL(generic_pipe_buf_get);
208
209 /**
210 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
211 * @pipe: the pipe that the buffer belongs to
212 * @buf: the buffer to put a reference to
213 *
214 * Description:
215 * This function releases a reference to @buf.
216 */
generic_pipe_buf_release(struct pipe_inode_info * pipe,struct pipe_buffer * buf)217 void generic_pipe_buf_release(struct pipe_inode_info *pipe,
218 struct pipe_buffer *buf)
219 {
220 put_page(buf->page);
221 }
222 EXPORT_SYMBOL(generic_pipe_buf_release);
223
224 static const struct pipe_buf_operations anon_pipe_buf_ops = {
225 .release = anon_pipe_buf_release,
226 .try_steal = anon_pipe_buf_try_steal,
227 .get = generic_pipe_buf_get,
228 };
229
230 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
pipe_readable(const struct pipe_inode_info * pipe)231 static inline bool pipe_readable(const struct pipe_inode_info *pipe)
232 {
233 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
234 unsigned int writers = READ_ONCE(pipe->writers);
235
236 return !pipe_empty(idx.head, idx.tail) || !writers;
237 }
238
pipe_update_tail(struct pipe_inode_info * pipe,struct pipe_buffer * buf,unsigned int tail)239 static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,
240 struct pipe_buffer *buf,
241 unsigned int tail)
242 {
243 pipe_buf_release(pipe, buf);
244
245 /*
246 * If the pipe has a watch_queue, we need additional protection
247 * by the spinlock because notifications get posted with only
248 * this spinlock, no mutex
249 */
250 if (pipe_has_watch_queue(pipe)) {
251 spin_lock_irq(&pipe->rd_wait.lock);
252 #ifdef CONFIG_WATCH_QUEUE
253 if (buf->flags & PIPE_BUF_FLAG_LOSS)
254 pipe->note_loss = true;
255 #endif
256 pipe->tail = ++tail;
257 spin_unlock_irq(&pipe->rd_wait.lock);
258 return tail;
259 }
260
261 /*
262 * Without a watch_queue, we can simply increment the tail
263 * without the spinlock - the mutex is enough.
264 */
265 pipe->tail = ++tail;
266 return tail;
267 }
268
269 static ssize_t
anon_pipe_read(struct kiocb * iocb,struct iov_iter * to)270 anon_pipe_read(struct kiocb *iocb, struct iov_iter *to)
271 {
272 size_t total_len = iov_iter_count(to);
273 struct file *filp = iocb->ki_filp;
274 struct pipe_inode_info *pipe = filp->private_data;
275 bool wake_writer = false, wake_next_reader = false;
276 ssize_t ret;
277
278 /* Null read succeeds. */
279 if (unlikely(total_len == 0))
280 return 0;
281
282 ret = 0;
283 mutex_lock(&pipe->mutex);
284
285 /*
286 * We only wake up writers if the pipe was full when we started reading
287 * and it is no longer full after reading to avoid unnecessary wakeups.
288 *
289 * But when we do wake up writers, we do so using a sync wakeup
290 * (WF_SYNC), because we want them to get going and generate more
291 * data for us.
292 */
293 for (;;) {
294 /* Read ->head with a barrier vs post_one_notification() */
295 unsigned int head = smp_load_acquire(&pipe->head);
296 unsigned int tail = pipe->tail;
297
298 #ifdef CONFIG_WATCH_QUEUE
299 if (pipe->note_loss) {
300 struct watch_notification n;
301
302 if (total_len < 8) {
303 if (ret == 0)
304 ret = -ENOBUFS;
305 break;
306 }
307
308 n.type = WATCH_TYPE_META;
309 n.subtype = WATCH_META_LOSS_NOTIFICATION;
310 n.info = watch_sizeof(n);
311 if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
312 if (ret == 0)
313 ret = -EFAULT;
314 break;
315 }
316 ret += sizeof(n);
317 total_len -= sizeof(n);
318 pipe->note_loss = false;
319 }
320 #endif
321
322 if (!pipe_empty(head, tail)) {
323 struct pipe_buffer *buf = pipe_buf(pipe, tail);
324 size_t chars = buf->len;
325 size_t written;
326 int error;
327
328 if (chars > total_len) {
329 if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
330 if (ret == 0)
331 ret = -ENOBUFS;
332 break;
333 }
334 chars = total_len;
335 }
336
337 error = pipe_buf_confirm(pipe, buf);
338 if (error) {
339 if (!ret)
340 ret = error;
341 break;
342 }
343
344 written = copy_page_to_iter(buf->page, buf->offset, chars, to);
345 if (unlikely(written < chars)) {
346 if (!ret)
347 ret = -EFAULT;
348 break;
349 }
350 ret += chars;
351 buf->offset += chars;
352 buf->len -= chars;
353
354 /* Was it a packet buffer? Clean up and exit */
355 if (buf->flags & PIPE_BUF_FLAG_PACKET) {
356 total_len = chars;
357 buf->len = 0;
358 }
359
360 if (!buf->len) {
361 wake_writer |= pipe_full(head, tail, pipe->max_usage);
362 tail = pipe_update_tail(pipe, buf, tail);
363 }
364 total_len -= chars;
365 if (!total_len)
366 break; /* common path: read succeeded */
367 if (!pipe_empty(head, tail)) /* More to do? */
368 continue;
369 }
370
371 if (!pipe->writers)
372 break;
373 if (ret)
374 break;
375 if ((filp->f_flags & O_NONBLOCK) ||
376 (iocb->ki_flags & IOCB_NOWAIT)) {
377 ret = -EAGAIN;
378 break;
379 }
380 mutex_unlock(&pipe->mutex);
381 /*
382 * We only get here if we didn't actually read anything.
383 *
384 * But because we didn't read anything, at this point we can
385 * just return directly with -ERESTARTSYS if we're interrupted,
386 * since we've done any required wakeups and there's no need
387 * to mark anything accessed. And we've dropped the lock.
388 */
389 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
390 return -ERESTARTSYS;
391
392 wake_next_reader = true;
393 mutex_lock(&pipe->mutex);
394 }
395 if (pipe_is_empty(pipe))
396 wake_next_reader = false;
397 mutex_unlock(&pipe->mutex);
398
399 if (wake_writer)
400 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
401 if (wake_next_reader)
402 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
403 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
404 return ret;
405 }
406
407 static ssize_t
fifo_pipe_read(struct kiocb * iocb,struct iov_iter * to)408 fifo_pipe_read(struct kiocb *iocb, struct iov_iter *to)
409 {
410 int ret = anon_pipe_read(iocb, to);
411 if (ret > 0)
412 file_accessed(iocb->ki_filp);
413 return ret;
414 }
415
is_packetized(struct file * file)416 static inline int is_packetized(struct file *file)
417 {
418 return (file->f_flags & O_DIRECT) != 0;
419 }
420
421 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
pipe_writable(const struct pipe_inode_info * pipe)422 static inline bool pipe_writable(const struct pipe_inode_info *pipe)
423 {
424 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
425 unsigned int max_usage = READ_ONCE(pipe->max_usage);
426
427 return !pipe_full(idx.head, idx.tail, max_usage) ||
428 !READ_ONCE(pipe->readers);
429 }
430
431 static ssize_t
anon_pipe_write(struct kiocb * iocb,struct iov_iter * from)432 anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
433 {
434 struct file *filp = iocb->ki_filp;
435 struct pipe_inode_info *pipe = filp->private_data;
436 unsigned int head;
437 ssize_t ret = 0;
438 size_t total_len = iov_iter_count(from);
439 ssize_t chars;
440 bool was_empty = false;
441 bool wake_next_writer = false;
442
443 /*
444 * Reject writing to watch queue pipes before the point where we lock
445 * the pipe.
446 * Otherwise, lockdep would be unhappy if the caller already has another
447 * pipe locked.
448 * If we had to support locking a normal pipe and a notification pipe at
449 * the same time, we could set up lockdep annotations for that, but
450 * since we don't actually need that, it's simpler to just bail here.
451 */
452 if (pipe_has_watch_queue(pipe))
453 return -EXDEV;
454
455 /* Null write succeeds. */
456 if (unlikely(total_len == 0))
457 return 0;
458
459 mutex_lock(&pipe->mutex);
460
461 if (!pipe->readers) {
462 send_sig(SIGPIPE, current, 0);
463 ret = -EPIPE;
464 goto out;
465 }
466
467 /*
468 * If it wasn't empty we try to merge new data into
469 * the last buffer.
470 *
471 * That naturally merges small writes, but it also
472 * page-aligns the rest of the writes for large writes
473 * spanning multiple pages.
474 */
475 head = pipe->head;
476 was_empty = pipe_empty(head, pipe->tail);
477 chars = total_len & (PAGE_SIZE-1);
478 if (chars && !was_empty) {
479 struct pipe_buffer *buf = pipe_buf(pipe, head - 1);
480 int offset = buf->offset + buf->len;
481
482 if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
483 offset + chars <= PAGE_SIZE) {
484 ret = pipe_buf_confirm(pipe, buf);
485 if (ret)
486 goto out;
487
488 ret = copy_page_from_iter(buf->page, offset, chars, from);
489 if (unlikely(ret < chars)) {
490 ret = -EFAULT;
491 goto out;
492 }
493
494 buf->len += ret;
495 if (!iov_iter_count(from))
496 goto out;
497 }
498 }
499
500 for (;;) {
501 if (!pipe->readers) {
502 send_sig(SIGPIPE, current, 0);
503 if (!ret)
504 ret = -EPIPE;
505 break;
506 }
507
508 head = pipe->head;
509 if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
510 struct pipe_buffer *buf;
511 struct page *page;
512 int copied;
513
514 page = anon_pipe_get_page(pipe);
515 if (unlikely(!page)) {
516 if (!ret)
517 ret = -ENOMEM;
518 break;
519 }
520
521 copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
522 if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
523 anon_pipe_put_page(pipe, page);
524 if (!ret)
525 ret = -EFAULT;
526 break;
527 }
528
529 pipe->head = head + 1;
530 /* Insert it into the buffer array */
531 buf = pipe_buf(pipe, head);
532 buf->page = page;
533 buf->ops = &anon_pipe_buf_ops;
534 buf->offset = 0;
535 if (is_packetized(filp))
536 buf->flags = PIPE_BUF_FLAG_PACKET;
537 else
538 buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
539
540 buf->len = copied;
541 ret += copied;
542
543 if (!iov_iter_count(from))
544 break;
545
546 continue;
547 }
548
549 /* Wait for buffer space to become available. */
550 if ((filp->f_flags & O_NONBLOCK) ||
551 (iocb->ki_flags & IOCB_NOWAIT)) {
552 if (!ret)
553 ret = -EAGAIN;
554 break;
555 }
556 if (signal_pending(current)) {
557 if (!ret)
558 ret = -ERESTARTSYS;
559 break;
560 }
561
562 /*
563 * We're going to release the pipe lock and wait for more
564 * space. We wake up any readers if necessary, and then
565 * after waiting we need to re-check whether the pipe
566 * become empty while we dropped the lock.
567 */
568 mutex_unlock(&pipe->mutex);
569 if (was_empty)
570 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
571 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
572 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
573 mutex_lock(&pipe->mutex);
574 was_empty = pipe_is_empty(pipe);
575 wake_next_writer = true;
576 }
577 out:
578 if (pipe_is_full(pipe))
579 wake_next_writer = false;
580 mutex_unlock(&pipe->mutex);
581
582 /*
583 * If we do do a wakeup event, we do a 'sync' wakeup, because we
584 * want the reader to start processing things asap, rather than
585 * leave the data pending.
586 *
587 * This is particularly important for small writes, because of
588 * how (for example) the GNU make jobserver uses small writes to
589 * wake up pending jobs
590 *
591 * Epoll nonsensically wants a wakeup whether the pipe
592 * was already empty or not.
593 */
594 if (was_empty || pipe->poll_usage)
595 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
596 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
597 if (wake_next_writer)
598 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
599 return ret;
600 }
601
602 static ssize_t
fifo_pipe_write(struct kiocb * iocb,struct iov_iter * from)603 fifo_pipe_write(struct kiocb *iocb, struct iov_iter *from)
604 {
605 int ret = anon_pipe_write(iocb, from);
606 if (ret > 0) {
607 struct file *filp = iocb->ki_filp;
608 if (sb_start_write_trylock(file_inode(filp)->i_sb)) {
609 int err = file_update_time(filp);
610 if (err)
611 ret = err;
612 sb_end_write(file_inode(filp)->i_sb);
613 }
614 }
615 return ret;
616 }
617
pipe_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)618 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
619 {
620 struct pipe_inode_info *pipe = filp->private_data;
621 unsigned int count, head, tail;
622
623 switch (cmd) {
624 case FIONREAD:
625 mutex_lock(&pipe->mutex);
626 count = 0;
627 head = pipe->head;
628 tail = pipe->tail;
629
630 while (!pipe_empty(head, tail)) {
631 count += pipe_buf(pipe, tail)->len;
632 tail++;
633 }
634 mutex_unlock(&pipe->mutex);
635
636 return put_user(count, (int __user *)arg);
637
638 #ifdef CONFIG_WATCH_QUEUE
639 case IOC_WATCH_QUEUE_SET_SIZE: {
640 int ret;
641 mutex_lock(&pipe->mutex);
642 ret = watch_queue_set_size(pipe, arg);
643 mutex_unlock(&pipe->mutex);
644 return ret;
645 }
646
647 case IOC_WATCH_QUEUE_SET_FILTER:
648 return watch_queue_set_filter(
649 pipe, (struct watch_notification_filter __user *)arg);
650 #endif
651
652 default:
653 return -ENOIOCTLCMD;
654 }
655 }
656
657 /* No kernel lock held - fine */
658 static __poll_t
pipe_poll(struct file * filp,poll_table * wait)659 pipe_poll(struct file *filp, poll_table *wait)
660 {
661 __poll_t mask;
662 struct pipe_inode_info *pipe = filp->private_data;
663 union pipe_index idx;
664
665 /* Epoll has some historical nasty semantics, this enables them */
666 WRITE_ONCE(pipe->poll_usage, true);
667
668 /*
669 * Reading pipe state only -- no need for acquiring the semaphore.
670 *
671 * But because this is racy, the code has to add the
672 * entry to the poll table _first_ ..
673 */
674 if (filp->f_mode & FMODE_READ)
675 poll_wait(filp, &pipe->rd_wait, wait);
676 if (filp->f_mode & FMODE_WRITE)
677 poll_wait(filp, &pipe->wr_wait, wait);
678
679 /*
680 * .. and only then can you do the racy tests. That way,
681 * if something changes and you got it wrong, the poll
682 * table entry will wake you up and fix it.
683 */
684 idx.head_tail = READ_ONCE(pipe->head_tail);
685
686 mask = 0;
687 if (filp->f_mode & FMODE_READ) {
688 if (!pipe_empty(idx.head, idx.tail))
689 mask |= EPOLLIN | EPOLLRDNORM;
690 if (!pipe->writers && filp->f_pipe != pipe->w_counter)
691 mask |= EPOLLHUP;
692 }
693
694 if (filp->f_mode & FMODE_WRITE) {
695 if (!pipe_full(idx.head, idx.tail, pipe->max_usage))
696 mask |= EPOLLOUT | EPOLLWRNORM;
697 /*
698 * Most Unices do not set EPOLLERR for FIFOs but on Linux they
699 * behave exactly like pipes for poll().
700 */
701 if (!pipe->readers)
702 mask |= EPOLLERR;
703 }
704
705 return mask;
706 }
707
put_pipe_info(struct inode * inode,struct pipe_inode_info * pipe)708 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
709 {
710 int kill = 0;
711
712 spin_lock(&inode->i_lock);
713 if (!--pipe->files) {
714 inode->i_pipe = NULL;
715 kill = 1;
716 }
717 spin_unlock(&inode->i_lock);
718
719 if (kill)
720 free_pipe_info(pipe);
721 }
722
723 static int
pipe_release(struct inode * inode,struct file * file)724 pipe_release(struct inode *inode, struct file *file)
725 {
726 struct pipe_inode_info *pipe = file->private_data;
727
728 mutex_lock(&pipe->mutex);
729 if (file->f_mode & FMODE_READ)
730 pipe->readers--;
731 if (file->f_mode & FMODE_WRITE)
732 pipe->writers--;
733
734 /* Was that the last reader or writer, but not the other side? */
735 if (!pipe->readers != !pipe->writers) {
736 wake_up_interruptible_all(&pipe->rd_wait);
737 wake_up_interruptible_all(&pipe->wr_wait);
738 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
739 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
740 }
741 mutex_unlock(&pipe->mutex);
742
743 put_pipe_info(inode, pipe);
744 return 0;
745 }
746
747 static int
pipe_fasync(int fd,struct file * filp,int on)748 pipe_fasync(int fd, struct file *filp, int on)
749 {
750 struct pipe_inode_info *pipe = filp->private_data;
751 int retval = 0;
752
753 mutex_lock(&pipe->mutex);
754 if (filp->f_mode & FMODE_READ)
755 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
756 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
757 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
758 if (retval < 0 && (filp->f_mode & FMODE_READ))
759 /* this can happen only if on == T */
760 fasync_helper(-1, filp, 0, &pipe->fasync_readers);
761 }
762 mutex_unlock(&pipe->mutex);
763 return retval;
764 }
765
account_pipe_buffers(struct user_struct * user,unsigned long old,unsigned long new)766 unsigned long account_pipe_buffers(struct user_struct *user,
767 unsigned long old, unsigned long new)
768 {
769 return atomic_long_add_return(new - old, &user->pipe_bufs);
770 }
771
too_many_pipe_buffers_soft(unsigned long user_bufs)772 bool too_many_pipe_buffers_soft(unsigned long user_bufs)
773 {
774 unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
775
776 return soft_limit && user_bufs > soft_limit;
777 }
778
too_many_pipe_buffers_hard(unsigned long user_bufs)779 bool too_many_pipe_buffers_hard(unsigned long user_bufs)
780 {
781 unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
782
783 return hard_limit && user_bufs > hard_limit;
784 }
785
pipe_is_unprivileged_user(void)786 bool pipe_is_unprivileged_user(void)
787 {
788 return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
789 }
790
alloc_pipe_info(void)791 struct pipe_inode_info *alloc_pipe_info(void)
792 {
793 struct pipe_inode_info *pipe;
794 unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
795 struct user_struct *user = get_current_user();
796 unsigned long user_bufs;
797 unsigned int max_size = READ_ONCE(pipe_max_size);
798
799 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
800 if (pipe == NULL)
801 goto out_free_uid;
802
803 if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
804 pipe_bufs = max_size >> PAGE_SHIFT;
805
806 user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
807
808 if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
809 user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
810 pipe_bufs = PIPE_MIN_DEF_BUFFERS;
811 }
812
813 if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
814 goto out_revert_acct;
815
816 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
817 GFP_KERNEL_ACCOUNT);
818
819 if (pipe->bufs) {
820 init_waitqueue_head(&pipe->rd_wait);
821 init_waitqueue_head(&pipe->wr_wait);
822 pipe->r_counter = pipe->w_counter = 1;
823 pipe->max_usage = pipe_bufs;
824 pipe->ring_size = pipe_bufs;
825 pipe->nr_accounted = pipe_bufs;
826 pipe->user = user;
827 mutex_init(&pipe->mutex);
828 lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL);
829 return pipe;
830 }
831
832 out_revert_acct:
833 (void) account_pipe_buffers(user, pipe_bufs, 0);
834 kfree(pipe);
835 out_free_uid:
836 free_uid(user);
837 return NULL;
838 }
839
free_pipe_info(struct pipe_inode_info * pipe)840 void free_pipe_info(struct pipe_inode_info *pipe)
841 {
842 unsigned int i;
843
844 #ifdef CONFIG_WATCH_QUEUE
845 if (pipe->watch_queue)
846 watch_queue_clear(pipe->watch_queue);
847 #endif
848
849 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
850 free_uid(pipe->user);
851 for (i = 0; i < pipe->ring_size; i++) {
852 struct pipe_buffer *buf = pipe->bufs + i;
853 if (buf->ops)
854 pipe_buf_release(pipe, buf);
855 }
856 #ifdef CONFIG_WATCH_QUEUE
857 if (pipe->watch_queue)
858 put_watch_queue(pipe->watch_queue);
859 #endif
860 for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
861 if (pipe->tmp_page[i])
862 __free_page(pipe->tmp_page[i]);
863 }
864 kfree(pipe->bufs);
865 kfree(pipe);
866 }
867
868 static struct vfsmount *pipe_mnt __ro_after_init;
869
870 /*
871 * pipefs_dname() is called from d_path().
872 */
pipefs_dname(struct dentry * dentry,char * buffer,int buflen)873 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
874 {
875 return dynamic_dname(buffer, buflen, "pipe:[%lu]",
876 d_inode(dentry)->i_ino);
877 }
878
879 static const struct dentry_operations pipefs_dentry_operations = {
880 .d_dname = pipefs_dname,
881 };
882
883 static const struct file_operations pipeanon_fops;
884
get_pipe_inode(void)885 static struct inode * get_pipe_inode(void)
886 {
887 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
888 struct pipe_inode_info *pipe;
889
890 if (!inode)
891 goto fail_inode;
892
893 inode->i_ino = get_next_ino();
894
895 pipe = alloc_pipe_info();
896 if (!pipe)
897 goto fail_iput;
898
899 inode->i_pipe = pipe;
900 pipe->files = 2;
901 pipe->readers = pipe->writers = 1;
902 inode->i_fop = &pipeanon_fops;
903
904 /*
905 * Mark the inode dirty from the very beginning,
906 * that way it will never be moved to the dirty
907 * list because "mark_inode_dirty()" will think
908 * that it already _is_ on the dirty list.
909 */
910 inode->i_state = I_DIRTY;
911 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
912 inode->i_uid = current_fsuid();
913 inode->i_gid = current_fsgid();
914 simple_inode_init_ts(inode);
915
916 return inode;
917
918 fail_iput:
919 iput(inode);
920
921 fail_inode:
922 return NULL;
923 }
924
create_pipe_files(struct file ** res,int flags)925 int create_pipe_files(struct file **res, int flags)
926 {
927 struct inode *inode = get_pipe_inode();
928 struct file *f;
929 int error;
930
931 if (!inode)
932 return -ENFILE;
933
934 if (flags & O_NOTIFICATION_PIPE) {
935 error = watch_queue_init(inode->i_pipe);
936 if (error) {
937 free_pipe_info(inode->i_pipe);
938 iput(inode);
939 return error;
940 }
941 }
942
943 f = alloc_file_pseudo(inode, pipe_mnt, "",
944 O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
945 &pipeanon_fops);
946 if (IS_ERR(f)) {
947 free_pipe_info(inode->i_pipe);
948 iput(inode);
949 return PTR_ERR(f);
950 }
951
952 f->private_data = inode->i_pipe;
953 f->f_pipe = 0;
954
955 res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),
956 &pipeanon_fops);
957 if (IS_ERR(res[0])) {
958 put_pipe_info(inode, inode->i_pipe);
959 fput(f);
960 return PTR_ERR(res[0]);
961 }
962 res[0]->private_data = inode->i_pipe;
963 res[0]->f_pipe = 0;
964 res[1] = f;
965 stream_open(inode, res[0]);
966 stream_open(inode, res[1]);
967 /*
968 * Disable permission and pre-content events, but enable legacy
969 * inotify events for legacy users.
970 */
971 file_set_fsnotify_mode(res[0], FMODE_NONOTIFY_PERM);
972 file_set_fsnotify_mode(res[1], FMODE_NONOTIFY_PERM);
973 return 0;
974 }
975
__do_pipe_flags(int * fd,struct file ** files,int flags)976 static int __do_pipe_flags(int *fd, struct file **files, int flags)
977 {
978 int error;
979 int fdw, fdr;
980
981 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
982 return -EINVAL;
983
984 error = create_pipe_files(files, flags);
985 if (error)
986 return error;
987
988 error = get_unused_fd_flags(flags);
989 if (error < 0)
990 goto err_read_pipe;
991 fdr = error;
992
993 error = get_unused_fd_flags(flags);
994 if (error < 0)
995 goto err_fdr;
996 fdw = error;
997
998 audit_fd_pair(fdr, fdw);
999 fd[0] = fdr;
1000 fd[1] = fdw;
1001 /* pipe groks IOCB_NOWAIT */
1002 files[0]->f_mode |= FMODE_NOWAIT;
1003 files[1]->f_mode |= FMODE_NOWAIT;
1004 return 0;
1005
1006 err_fdr:
1007 put_unused_fd(fdr);
1008 err_read_pipe:
1009 fput(files[0]);
1010 fput(files[1]);
1011 return error;
1012 }
1013
do_pipe_flags(int * fd,int flags)1014 int do_pipe_flags(int *fd, int flags)
1015 {
1016 struct file *files[2];
1017 int error = __do_pipe_flags(fd, files, flags);
1018 if (!error) {
1019 fd_install(fd[0], files[0]);
1020 fd_install(fd[1], files[1]);
1021 }
1022 return error;
1023 }
1024
1025 /*
1026 * sys_pipe() is the normal C calling standard for creating
1027 * a pipe. It's not the way Unix traditionally does this, though.
1028 */
do_pipe2(int __user * fildes,int flags)1029 static int do_pipe2(int __user *fildes, int flags)
1030 {
1031 struct file *files[2];
1032 int fd[2];
1033 int error;
1034
1035 error = __do_pipe_flags(fd, files, flags);
1036 if (!error) {
1037 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
1038 fput(files[0]);
1039 fput(files[1]);
1040 put_unused_fd(fd[0]);
1041 put_unused_fd(fd[1]);
1042 error = -EFAULT;
1043 } else {
1044 fd_install(fd[0], files[0]);
1045 fd_install(fd[1], files[1]);
1046 }
1047 }
1048 return error;
1049 }
1050
SYSCALL_DEFINE2(pipe2,int __user *,fildes,int,flags)1051 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
1052 {
1053 return do_pipe2(fildes, flags);
1054 }
1055
SYSCALL_DEFINE1(pipe,int __user *,fildes)1056 SYSCALL_DEFINE1(pipe, int __user *, fildes)
1057 {
1058 return do_pipe2(fildes, 0);
1059 }
1060
1061 /*
1062 * This is the stupid "wait for pipe to be readable or writable"
1063 * model.
1064 *
1065 * See pipe_read/write() for the proper kind of exclusive wait,
1066 * but that requires that we wake up any other readers/writers
1067 * if we then do not end up reading everything (ie the whole
1068 * "wake_next_reader/writer" logic in pipe_read/write()).
1069 */
pipe_wait_readable(struct pipe_inode_info * pipe)1070 void pipe_wait_readable(struct pipe_inode_info *pipe)
1071 {
1072 pipe_unlock(pipe);
1073 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
1074 pipe_lock(pipe);
1075 }
1076
pipe_wait_writable(struct pipe_inode_info * pipe)1077 void pipe_wait_writable(struct pipe_inode_info *pipe)
1078 {
1079 pipe_unlock(pipe);
1080 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
1081 pipe_lock(pipe);
1082 }
1083
1084 /*
1085 * This depends on both the wait (here) and the wakeup (wake_up_partner)
1086 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1087 * race with the count check and waitqueue prep.
1088 *
1089 * Normally in order to avoid races, you'd do the prepare_to_wait() first,
1090 * then check the condition you're waiting for, and only then sleep. But
1091 * because of the pipe lock, we can check the condition before being on
1092 * the wait queue.
1093 *
1094 * We use the 'rd_wait' waitqueue for pipe partner waiting.
1095 */
wait_for_partner(struct pipe_inode_info * pipe,unsigned int * cnt)1096 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
1097 {
1098 DEFINE_WAIT(rdwait);
1099 int cur = *cnt;
1100
1101 while (cur == *cnt) {
1102 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
1103 pipe_unlock(pipe);
1104 schedule();
1105 finish_wait(&pipe->rd_wait, &rdwait);
1106 pipe_lock(pipe);
1107 if (signal_pending(current))
1108 break;
1109 }
1110 return cur == *cnt ? -ERESTARTSYS : 0;
1111 }
1112
wake_up_partner(struct pipe_inode_info * pipe)1113 static void wake_up_partner(struct pipe_inode_info *pipe)
1114 {
1115 wake_up_interruptible_all(&pipe->rd_wait);
1116 }
1117
fifo_open(struct inode * inode,struct file * filp)1118 static int fifo_open(struct inode *inode, struct file *filp)
1119 {
1120 bool is_pipe = inode->i_fop == &pipeanon_fops;
1121 struct pipe_inode_info *pipe;
1122 int ret;
1123
1124 filp->f_pipe = 0;
1125
1126 spin_lock(&inode->i_lock);
1127 if (inode->i_pipe) {
1128 pipe = inode->i_pipe;
1129 pipe->files++;
1130 spin_unlock(&inode->i_lock);
1131 } else {
1132 spin_unlock(&inode->i_lock);
1133 pipe = alloc_pipe_info();
1134 if (!pipe)
1135 return -ENOMEM;
1136 pipe->files = 1;
1137 spin_lock(&inode->i_lock);
1138 if (unlikely(inode->i_pipe)) {
1139 inode->i_pipe->files++;
1140 spin_unlock(&inode->i_lock);
1141 free_pipe_info(pipe);
1142 pipe = inode->i_pipe;
1143 } else {
1144 inode->i_pipe = pipe;
1145 spin_unlock(&inode->i_lock);
1146 }
1147 }
1148 filp->private_data = pipe;
1149 /* OK, we have a pipe and it's pinned down */
1150
1151 mutex_lock(&pipe->mutex);
1152
1153 /* We can only do regular read/write on fifos */
1154 stream_open(inode, filp);
1155
1156 switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {
1157 case FMODE_READ:
1158 /*
1159 * O_RDONLY
1160 * POSIX.1 says that O_NONBLOCK means return with the FIFO
1161 * opened, even when there is no process writing the FIFO.
1162 */
1163 pipe->r_counter++;
1164 if (pipe->readers++ == 0)
1165 wake_up_partner(pipe);
1166
1167 if (!is_pipe && !pipe->writers) {
1168 if ((filp->f_flags & O_NONBLOCK)) {
1169 /* suppress EPOLLHUP until we have
1170 * seen a writer */
1171 filp->f_pipe = pipe->w_counter;
1172 } else {
1173 if (wait_for_partner(pipe, &pipe->w_counter))
1174 goto err_rd;
1175 }
1176 }
1177 break;
1178
1179 case FMODE_WRITE:
1180 /*
1181 * O_WRONLY
1182 * POSIX.1 says that O_NONBLOCK means return -1 with
1183 * errno=ENXIO when there is no process reading the FIFO.
1184 */
1185 ret = -ENXIO;
1186 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
1187 goto err;
1188
1189 pipe->w_counter++;
1190 if (!pipe->writers++)
1191 wake_up_partner(pipe);
1192
1193 if (!is_pipe && !pipe->readers) {
1194 if (wait_for_partner(pipe, &pipe->r_counter))
1195 goto err_wr;
1196 }
1197 break;
1198
1199 case FMODE_READ | FMODE_WRITE:
1200 /*
1201 * O_RDWR
1202 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
1203 * This implementation will NEVER block on a O_RDWR open, since
1204 * the process can at least talk to itself.
1205 */
1206
1207 pipe->readers++;
1208 pipe->writers++;
1209 pipe->r_counter++;
1210 pipe->w_counter++;
1211 if (pipe->readers == 1 || pipe->writers == 1)
1212 wake_up_partner(pipe);
1213 break;
1214
1215 default:
1216 ret = -EINVAL;
1217 goto err;
1218 }
1219
1220 /* Ok! */
1221 mutex_unlock(&pipe->mutex);
1222 return 0;
1223
1224 err_rd:
1225 if (!--pipe->readers)
1226 wake_up_interruptible(&pipe->wr_wait);
1227 ret = -ERESTARTSYS;
1228 goto err;
1229
1230 err_wr:
1231 if (!--pipe->writers)
1232 wake_up_interruptible_all(&pipe->rd_wait);
1233 ret = -ERESTARTSYS;
1234 goto err;
1235
1236 err:
1237 mutex_unlock(&pipe->mutex);
1238
1239 put_pipe_info(inode, pipe);
1240 return ret;
1241 }
1242
1243 const struct file_operations pipefifo_fops = {
1244 .open = fifo_open,
1245 .read_iter = fifo_pipe_read,
1246 .write_iter = fifo_pipe_write,
1247 .poll = pipe_poll,
1248 .unlocked_ioctl = pipe_ioctl,
1249 .release = pipe_release,
1250 .fasync = pipe_fasync,
1251 .splice_write = iter_file_splice_write,
1252 };
1253
1254 static const struct file_operations pipeanon_fops = {
1255 .open = fifo_open,
1256 .read_iter = anon_pipe_read,
1257 .write_iter = anon_pipe_write,
1258 .poll = pipe_poll,
1259 .unlocked_ioctl = pipe_ioctl,
1260 .release = pipe_release,
1261 .fasync = pipe_fasync,
1262 .splice_write = iter_file_splice_write,
1263 };
1264
1265 /*
1266 * Currently we rely on the pipe array holding a power-of-2 number
1267 * of pages. Returns 0 on error.
1268 */
round_pipe_size(unsigned int size)1269 unsigned int round_pipe_size(unsigned int size)
1270 {
1271 if (size > (1U << 31))
1272 return 0;
1273
1274 /* Minimum pipe size, as required by POSIX */
1275 if (size < PAGE_SIZE)
1276 return PAGE_SIZE;
1277
1278 return roundup_pow_of_two(size);
1279 }
1280
1281 /*
1282 * Resize the pipe ring to a number of slots.
1283 *
1284 * Note the pipe can be reduced in capacity, but only if the current
1285 * occupancy doesn't exceed nr_slots; if it does, EBUSY will be
1286 * returned instead.
1287 */
pipe_resize_ring(struct pipe_inode_info * pipe,unsigned int nr_slots)1288 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
1289 {
1290 struct pipe_buffer *bufs;
1291 unsigned int head, tail, mask, n;
1292
1293 /* nr_slots larger than limits of pipe->{head,tail} */
1294 if (unlikely(nr_slots > (pipe_index_t)-1u))
1295 return -EINVAL;
1296
1297 bufs = kcalloc(nr_slots, sizeof(*bufs),
1298 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
1299 if (unlikely(!bufs))
1300 return -ENOMEM;
1301
1302 spin_lock_irq(&pipe->rd_wait.lock);
1303 mask = pipe->ring_size - 1;
1304 head = pipe->head;
1305 tail = pipe->tail;
1306
1307 n = pipe_occupancy(head, tail);
1308 if (nr_slots < n) {
1309 spin_unlock_irq(&pipe->rd_wait.lock);
1310 kfree(bufs);
1311 return -EBUSY;
1312 }
1313
1314 /*
1315 * The pipe array wraps around, so just start the new one at zero
1316 * and adjust the indices.
1317 */
1318 if (n > 0) {
1319 unsigned int h = head & mask;
1320 unsigned int t = tail & mask;
1321 if (h > t) {
1322 memcpy(bufs, pipe->bufs + t,
1323 n * sizeof(struct pipe_buffer));
1324 } else {
1325 unsigned int tsize = pipe->ring_size - t;
1326 if (h > 0)
1327 memcpy(bufs + tsize, pipe->bufs,
1328 h * sizeof(struct pipe_buffer));
1329 memcpy(bufs, pipe->bufs + t,
1330 tsize * sizeof(struct pipe_buffer));
1331 }
1332 }
1333
1334 head = n;
1335 tail = 0;
1336
1337 kfree(pipe->bufs);
1338 pipe->bufs = bufs;
1339 pipe->ring_size = nr_slots;
1340 if (pipe->max_usage > nr_slots)
1341 pipe->max_usage = nr_slots;
1342 pipe->tail = tail;
1343 pipe->head = head;
1344
1345 if (!pipe_has_watch_queue(pipe)) {
1346 pipe->max_usage = nr_slots;
1347 pipe->nr_accounted = nr_slots;
1348 }
1349
1350 spin_unlock_irq(&pipe->rd_wait.lock);
1351
1352 /* This might have made more room for writers */
1353 wake_up_interruptible(&pipe->wr_wait);
1354 return 0;
1355 }
1356
1357 /*
1358 * Allocate a new array of pipe buffers and copy the info over. Returns the
1359 * pipe size if successful, or return -ERROR on error.
1360 */
pipe_set_size(struct pipe_inode_info * pipe,unsigned int arg)1361 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg)
1362 {
1363 unsigned long user_bufs;
1364 unsigned int nr_slots, size;
1365 long ret = 0;
1366
1367 if (pipe_has_watch_queue(pipe))
1368 return -EBUSY;
1369
1370 size = round_pipe_size(arg);
1371 nr_slots = size >> PAGE_SHIFT;
1372
1373 if (!nr_slots)
1374 return -EINVAL;
1375
1376 /*
1377 * If trying to increase the pipe capacity, check that an
1378 * unprivileged user is not trying to exceed various limits
1379 * (soft limit check here, hard limit check just below).
1380 * Decreasing the pipe capacity is always permitted, even
1381 * if the user is currently over a limit.
1382 */
1383 if (nr_slots > pipe->max_usage &&
1384 size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
1385 return -EPERM;
1386
1387 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
1388
1389 if (nr_slots > pipe->max_usage &&
1390 (too_many_pipe_buffers_hard(user_bufs) ||
1391 too_many_pipe_buffers_soft(user_bufs)) &&
1392 pipe_is_unprivileged_user()) {
1393 ret = -EPERM;
1394 goto out_revert_acct;
1395 }
1396
1397 ret = pipe_resize_ring(pipe, nr_slots);
1398 if (ret < 0)
1399 goto out_revert_acct;
1400
1401 return pipe->max_usage * PAGE_SIZE;
1402
1403 out_revert_acct:
1404 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
1405 return ret;
1406 }
1407
1408 /*
1409 * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is
1410 * not enough to verify that this is a pipe.
1411 */
get_pipe_info(struct file * file,bool for_splice)1412 struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
1413 {
1414 struct pipe_inode_info *pipe = file->private_data;
1415
1416 if (!pipe)
1417 return NULL;
1418 if (file->f_op != &pipefifo_fops && file->f_op != &pipeanon_fops)
1419 return NULL;
1420 if (for_splice && pipe_has_watch_queue(pipe))
1421 return NULL;
1422 return pipe;
1423 }
1424
pipe_fcntl(struct file * file,unsigned int cmd,unsigned int arg)1425 long pipe_fcntl(struct file *file, unsigned int cmd, unsigned int arg)
1426 {
1427 struct pipe_inode_info *pipe;
1428 long ret;
1429
1430 pipe = get_pipe_info(file, false);
1431 if (!pipe)
1432 return -EBADF;
1433
1434 mutex_lock(&pipe->mutex);
1435
1436 switch (cmd) {
1437 case F_SETPIPE_SZ:
1438 ret = pipe_set_size(pipe, arg);
1439 break;
1440 case F_GETPIPE_SZ:
1441 ret = pipe->max_usage * PAGE_SIZE;
1442 break;
1443 default:
1444 ret = -EINVAL;
1445 break;
1446 }
1447
1448 mutex_unlock(&pipe->mutex);
1449 return ret;
1450 }
1451
1452 static const struct super_operations pipefs_ops = {
1453 .destroy_inode = free_inode_nonrcu,
1454 .statfs = simple_statfs,
1455 };
1456
1457 /*
1458 * pipefs should _never_ be mounted by userland - too much of security hassle,
1459 * no real gain from having the whole file system mounted. So we don't need
1460 * any operations on the root directory. However, we need a non-trivial
1461 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1462 */
1463
pipefs_init_fs_context(struct fs_context * fc)1464 static int pipefs_init_fs_context(struct fs_context *fc)
1465 {
1466 struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
1467 if (!ctx)
1468 return -ENOMEM;
1469 ctx->ops = &pipefs_ops;
1470 ctx->dops = &pipefs_dentry_operations;
1471 return 0;
1472 }
1473
1474 static struct file_system_type pipe_fs_type = {
1475 .name = "pipefs",
1476 .init_fs_context = pipefs_init_fs_context,
1477 .kill_sb = kill_anon_super,
1478 };
1479
1480 #ifdef CONFIG_SYSCTL
do_proc_dopipe_max_size_conv(unsigned long * lvalp,unsigned int * valp,int write,void * data)1481 static int do_proc_dopipe_max_size_conv(unsigned long *lvalp,
1482 unsigned int *valp,
1483 int write, void *data)
1484 {
1485 if (write) {
1486 unsigned int val;
1487
1488 val = round_pipe_size(*lvalp);
1489 if (val == 0)
1490 return -EINVAL;
1491
1492 *valp = val;
1493 } else {
1494 unsigned int val = *valp;
1495 *lvalp = (unsigned long) val;
1496 }
1497
1498 return 0;
1499 }
1500
proc_dopipe_max_size(const struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)1501 static int proc_dopipe_max_size(const struct ctl_table *table, int write,
1502 void *buffer, size_t *lenp, loff_t *ppos)
1503 {
1504 return do_proc_douintvec(table, write, buffer, lenp, ppos,
1505 do_proc_dopipe_max_size_conv, NULL);
1506 }
1507
1508 static const struct ctl_table fs_pipe_sysctls[] = {
1509 {
1510 .procname = "pipe-max-size",
1511 .data = &pipe_max_size,
1512 .maxlen = sizeof(pipe_max_size),
1513 .mode = 0644,
1514 .proc_handler = proc_dopipe_max_size,
1515 },
1516 {
1517 .procname = "pipe-user-pages-hard",
1518 .data = &pipe_user_pages_hard,
1519 .maxlen = sizeof(pipe_user_pages_hard),
1520 .mode = 0644,
1521 .proc_handler = proc_doulongvec_minmax,
1522 },
1523 {
1524 .procname = "pipe-user-pages-soft",
1525 .data = &pipe_user_pages_soft,
1526 .maxlen = sizeof(pipe_user_pages_soft),
1527 .mode = 0644,
1528 .proc_handler = proc_doulongvec_minmax,
1529 },
1530 };
1531 #endif
1532
init_pipe_fs(void)1533 static int __init init_pipe_fs(void)
1534 {
1535 int err = register_filesystem(&pipe_fs_type);
1536
1537 if (!err) {
1538 pipe_mnt = kern_mount(&pipe_fs_type);
1539 if (IS_ERR(pipe_mnt)) {
1540 err = PTR_ERR(pipe_mnt);
1541 unregister_filesystem(&pipe_fs_type);
1542 }
1543 }
1544 #ifdef CONFIG_SYSCTL
1545 register_sysctl_init("fs", fs_pipe_sysctls);
1546 #endif
1547 return err;
1548 }
1549
1550 fs_initcall(init_pipe_fs);
1551