1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/fs/pipe.c 4 * 5 * Copyright (C) 1991, 1992, 1999 Linus Torvalds 6 */ 7 8 #include <linux/mm.h> 9 #include <linux/file.h> 10 #include <linux/poll.h> 11 #include <linux/slab.h> 12 #include <linux/module.h> 13 #include <linux/init.h> 14 #include <linux/fs.h> 15 #include <linux/log2.h> 16 #include <linux/mount.h> 17 #include <linux/pseudo_fs.h> 18 #include <linux/magic.h> 19 #include <linux/pipe_fs_i.h> 20 #include <linux/uio.h> 21 #include <linux/highmem.h> 22 #include <linux/pagemap.h> 23 #include <linux/audit.h> 24 #include <linux/syscalls.h> 25 #include <linux/fcntl.h> 26 #include <linux/memcontrol.h> 27 #include <linux/watch_queue.h> 28 #include <linux/sysctl.h> 29 #include <linux/sort.h> 30 31 #include <linux/uaccess.h> 32 #include <asm/ioctls.h> 33 34 #include "internal.h" 35 36 /* 37 * New pipe buffers will be restricted to this size while the user is exceeding 38 * their pipe buffer quota. The general pipe use case needs at least two 39 * buffers: one for data yet to be read, and one for new data. If this is less 40 * than two, then a write to a non-empty pipe may block even if the pipe is not 41 * full. This can occur with GNU make jobserver or similar uses of pipes as 42 * semaphores: multiple processes may be waiting to write tokens back to the 43 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/. 44 * 45 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their 46 * own risk, namely: pipe writes to non-full pipes may block until the pipe is 47 * emptied. 48 */ 49 #define PIPE_MIN_DEF_BUFFERS 2 50 51 /* 52 * The max size that a non-root user is allowed to grow the pipe. Can 53 * be set by root in /proc/sys/fs/pipe-max-size 54 */ 55 static unsigned int pipe_max_size = 1048576; 56 57 /* Maximum allocatable pages per user. Hard limit is unset by default, soft 58 * matches default values. 59 */ 60 static unsigned long pipe_user_pages_hard; 61 static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; 62 63 /* 64 * We use head and tail indices that aren't masked off, except at the point of 65 * dereference, but rather they're allowed to wrap naturally. This means there 66 * isn't a dead spot in the buffer, but the ring has to be a power of two and 67 * <= 2^31. 68 * -- David Howells 2019-09-23. 69 * 70 * Reads with count = 0 should always return 0. 71 * -- Julian Bradfield 1999-06-07. 72 * 73 * FIFOs and Pipes now generate SIGIO for both readers and writers. 74 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 75 * 76 * pipe_read & write cleanup 77 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 78 */ 79 80 #ifdef CONFIG_PROVE_LOCKING 81 static int pipe_lock_cmp_fn(const struct lockdep_map *a, 82 const struct lockdep_map *b) 83 { 84 return cmp_int((unsigned long) a, (unsigned long) b); 85 } 86 #endif 87 88 void pipe_lock(struct pipe_inode_info *pipe) 89 { 90 if (pipe->files) 91 mutex_lock(&pipe->mutex); 92 } 93 EXPORT_SYMBOL(pipe_lock); 94 95 void pipe_unlock(struct pipe_inode_info *pipe) 96 { 97 if (pipe->files) 98 mutex_unlock(&pipe->mutex); 99 } 100 EXPORT_SYMBOL(pipe_unlock); 101 102 void pipe_double_lock(struct pipe_inode_info *pipe1, 103 struct pipe_inode_info *pipe2) 104 { 105 BUG_ON(pipe1 == pipe2); 106 107 if (pipe1 > pipe2) 108 swap(pipe1, pipe2); 109 110 pipe_lock(pipe1); 111 pipe_lock(pipe2); 112 } 113 114 static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc) 115 { 116 if (!prealloc->count) 117 return NULL; 118 119 prealloc->count--; 120 121 return prealloc->pages[prealloc->count]; 122 } 123 124 /* Push a page to the prealloc pool. Returns true if added, false if full. */ 125 static bool anon_pipe_prealloc_push(struct anon_pipe_prealloc *prealloc, 126 struct page *page) 127 { 128 if (prealloc->count >= PIPE_PREALLOC_MAX) 129 return false; 130 prealloc->pages[prealloc->count++] = page; 131 return true; 132 } 133 134 /* 135 * Top up the pipe's own pool, then take pipe->mutex and return with it held. 136 * The shortfall is allocated outside the lock; the push and the caller's write 137 * then run under a single lock acquisition, avoiding a separate prefill 138 * lock/unlock cycle. anon_pipe_get_page() drains the pool instead of allocating 139 * under the lock. 140 */ 141 static void anon_pipe_prefill_and_lock(struct pipe_inode_info *pipe, size_t total_len) 142 { 143 struct page *pages[PIPE_PREALLOC_MAX]; 144 unsigned int want, have, need, n = 0; 145 146 want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE), 147 PIPE_PREALLOC_MAX); 148 /* Unlocked read; the pool is refilled under the lock below. */ 149 have = min_t(unsigned int, READ_ONCE(pipe->prealloc.count), want); 150 need = want - have; 151 152 if (!need) { 153 mutex_lock(&pipe->mutex); 154 return; 155 } 156 157 while (n < need) { 158 struct page *page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT); 159 160 if (!page) 161 break; 162 pages[n++] = page; 163 } 164 165 mutex_lock(&pipe->mutex); 166 while (n && anon_pipe_prealloc_push(&pipe->prealloc, pages[n - 1])) 167 n--; 168 169 /* 170 * Just flush any extra page that got affected by the TOCTOU 171 * effect 172 */ 173 while (n) 174 put_page(pages[--n]); 175 } 176 177 /* 178 * Called with pipe->mutex held. Trim the pool down to PIPE_PREALLOC_KEEP under 179 * the lock, drop it, then free the excess outside the critical section. 180 */ 181 static void anon_pipe_trim_and_unlock(struct pipe_inode_info *pipe) 182 { 183 struct page *excess[PIPE_PREALLOC_MAX]; 184 unsigned int nexcess = 0; 185 186 while (pipe->prealloc.count > PIPE_PREALLOC_KEEP) 187 excess[nexcess++] = anon_pipe_prealloc_pop(&pipe->prealloc); 188 mutex_unlock(&pipe->mutex); 189 190 while (nexcess) 191 put_page(excess[--nexcess]); 192 } 193 194 static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe) 195 { 196 struct page *page; 197 198 /* Drain the prealloc pool before allocating. Called with mutex held. */ 199 page = anon_pipe_prealloc_pop(&pipe->prealloc); 200 if (page) 201 return page; 202 203 /* FWIW: This is called with pipe->mutex held */ 204 return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT); 205 } 206 207 static void anon_pipe_put_page(struct pipe_inode_info *pipe, 208 struct page *page) 209 { 210 if (page_count(page) == 1 && 211 anon_pipe_prealloc_push(&pipe->prealloc, page)) 212 return; 213 214 put_page(page); 215 } 216 217 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, 218 struct pipe_buffer *buf) 219 { 220 struct page *page = buf->page; 221 222 anon_pipe_put_page(pipe, page); 223 } 224 225 /** 226 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer 227 * @pipe: the pipe that the buffer belongs to 228 * @buf: the buffer to get a reference to 229 * 230 * Description: 231 * This function grabs an extra reference to @buf. It's used in 232 * the tee() system call, when we duplicate the buffers in one 233 * pipe into another. 234 */ 235 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 236 { 237 return try_get_page(buf->page); 238 } 239 EXPORT_SYMBOL(generic_pipe_buf_get); 240 241 /** 242 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer 243 * @pipe: the pipe that the buffer belongs to 244 * @buf: the buffer to put a reference to 245 * 246 * Description: 247 * This function releases a reference to @buf. 248 */ 249 void generic_pipe_buf_release(struct pipe_inode_info *pipe, 250 struct pipe_buffer *buf) 251 { 252 put_page(buf->page); 253 } 254 EXPORT_SYMBOL(generic_pipe_buf_release); 255 256 static const struct pipe_buf_operations anon_pipe_buf_ops = { 257 .release = anon_pipe_buf_release, 258 .get = generic_pipe_buf_get, 259 }; 260 261 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */ 262 static inline bool pipe_readable(const struct pipe_inode_info *pipe) 263 { 264 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; 265 unsigned int writers = READ_ONCE(pipe->writers); 266 267 return !pipe_empty(idx.head, idx.tail) || !writers; 268 } 269 270 static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe, 271 struct pipe_buffer *buf, 272 unsigned int tail) 273 { 274 pipe_buf_release(pipe, buf); 275 276 /* 277 * If the pipe has a watch_queue, we need additional protection 278 * by the spinlock because notifications get posted with only 279 * this spinlock, no mutex 280 */ 281 if (pipe_has_watch_queue(pipe)) { 282 spin_lock_irq(&pipe->rd_wait.lock); 283 #ifdef CONFIG_WATCH_QUEUE 284 if (buf->flags & PIPE_BUF_FLAG_LOSS) 285 pipe->note_loss = true; 286 #endif 287 pipe->tail = ++tail; 288 spin_unlock_irq(&pipe->rd_wait.lock); 289 return tail; 290 } 291 292 /* 293 * Without a watch_queue, we can simply increment the tail 294 * without the spinlock - the mutex is enough. 295 */ 296 pipe->tail = ++tail; 297 return tail; 298 } 299 300 static ssize_t 301 anon_pipe_read(struct kiocb *iocb, struct iov_iter *to) 302 { 303 size_t total_len = iov_iter_count(to); 304 struct file *filp = iocb->ki_filp; 305 struct pipe_inode_info *pipe = filp->private_data; 306 bool wake_writer = false, wake_next_reader = false; 307 ssize_t ret; 308 309 /* Null read succeeds. */ 310 if (unlikely(total_len == 0)) 311 return 0; 312 313 ret = 0; 314 mutex_lock(&pipe->mutex); 315 316 /* 317 * We only wake up writers if the pipe was full when we started reading 318 * and it is no longer full after reading to avoid unnecessary wakeups. 319 * 320 * But when we do wake up writers, we do so using a sync wakeup 321 * (WF_SYNC), because we want them to get going and generate more 322 * data for us. 323 */ 324 for (;;) { 325 /* Read ->head with a barrier vs post_one_notification() */ 326 unsigned int head = smp_load_acquire(&pipe->head); 327 unsigned int tail = pipe->tail; 328 329 #ifdef CONFIG_WATCH_QUEUE 330 if (pipe->note_loss) { 331 struct watch_notification n; 332 333 if (total_len < 8) { 334 if (ret == 0) 335 ret = -ENOBUFS; 336 break; 337 } 338 339 n.type = WATCH_TYPE_META; 340 n.subtype = WATCH_META_LOSS_NOTIFICATION; 341 n.info = watch_sizeof(n); 342 if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) { 343 if (ret == 0) 344 ret = -EFAULT; 345 break; 346 } 347 ret += sizeof(n); 348 total_len -= sizeof(n); 349 pipe->note_loss = false; 350 } 351 #endif 352 353 if (!pipe_empty(head, tail)) { 354 struct pipe_buffer *buf = pipe_buf(pipe, tail); 355 size_t chars = buf->len; 356 size_t written; 357 int error; 358 359 if (chars > total_len) { 360 if (buf->flags & PIPE_BUF_FLAG_WHOLE) { 361 if (ret == 0) 362 ret = -ENOBUFS; 363 break; 364 } 365 chars = total_len; 366 } 367 368 error = pipe_buf_confirm(pipe, buf); 369 if (error) { 370 if (!ret) 371 ret = error; 372 break; 373 } 374 375 written = copy_page_to_iter(buf->page, buf->offset, chars, to); 376 if (unlikely(written < chars)) { 377 if (!ret) 378 ret = -EFAULT; 379 break; 380 } 381 ret += chars; 382 buf->offset += chars; 383 buf->len -= chars; 384 385 /* Was it a packet buffer? Clean up and exit */ 386 if (buf->flags & PIPE_BUF_FLAG_PACKET) { 387 total_len = chars; 388 buf->len = 0; 389 } 390 391 if (!buf->len) { 392 wake_writer |= pipe_full(head, tail, pipe->max_usage); 393 tail = pipe_update_tail(pipe, buf, tail); 394 } 395 total_len -= chars; 396 if (!total_len) 397 break; /* common path: read succeeded */ 398 if (!pipe_empty(head, tail)) /* More to do? */ 399 continue; 400 } 401 402 if (!pipe->writers) 403 break; 404 if (ret) 405 break; 406 if ((filp->f_flags & O_NONBLOCK) || 407 (iocb->ki_flags & IOCB_NOWAIT)) { 408 ret = -EAGAIN; 409 break; 410 } 411 mutex_unlock(&pipe->mutex); 412 /* 413 * We only get here if we didn't actually read anything. 414 * 415 * But because we didn't read anything, at this point we can 416 * just return directly with -ERESTARTSYS if we're interrupted, 417 * since we've done any required wakeups and there's no need 418 * to mark anything accessed. And we've dropped the lock. 419 */ 420 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0) 421 return -ERESTARTSYS; 422 423 wake_next_reader = true; 424 mutex_lock(&pipe->mutex); 425 } 426 if (pipe_is_empty(pipe)) 427 wake_next_reader = false; 428 /* Consumed buffers may have refilled the pool; trim it and unlock. */ 429 anon_pipe_trim_and_unlock(pipe); 430 431 if (wake_writer) 432 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); 433 if (wake_next_reader) 434 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); 435 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 436 return ret; 437 } 438 439 static ssize_t 440 fifo_pipe_read(struct kiocb *iocb, struct iov_iter *to) 441 { 442 int ret = anon_pipe_read(iocb, to); 443 if (ret > 0) 444 file_accessed(iocb->ki_filp); 445 return ret; 446 } 447 448 static inline int is_packetized(struct file *file) 449 { 450 return (file->f_flags & O_DIRECT) != 0; 451 } 452 453 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */ 454 static inline bool pipe_writable(const struct pipe_inode_info *pipe) 455 { 456 union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; 457 unsigned int max_usage = READ_ONCE(pipe->max_usage); 458 459 return !pipe_full(idx.head, idx.tail, max_usage) || 460 !READ_ONCE(pipe->readers); 461 } 462 463 static ssize_t 464 anon_pipe_write(struct kiocb *iocb, struct iov_iter *from) 465 { 466 struct file *filp = iocb->ki_filp; 467 struct pipe_inode_info *pipe = filp->private_data; 468 unsigned int head; 469 ssize_t ret = 0; 470 size_t total_len = iov_iter_count(from); 471 ssize_t chars; 472 bool was_empty = false; 473 bool wake_next_writer = false; 474 475 /* 476 * Reject writing to watch queue pipes before the point where we lock 477 * the pipe. 478 * Otherwise, lockdep would be unhappy if the caller already has another 479 * pipe locked. 480 * If we had to support locking a normal pipe and a notification pipe at 481 * the same time, we could set up lockdep annotations for that, but 482 * since we don't actually need that, it's simpler to just bail here. 483 */ 484 if (pipe_has_watch_queue(pipe)) 485 return -EXDEV; 486 487 /* Null write succeeds. */ 488 if (unlikely(total_len == 0)) 489 return 0; 490 491 anon_pipe_prefill_and_lock(pipe, total_len); 492 493 if (!pipe->readers) { 494 if ((iocb->ki_flags & IOCB_NOSIGNAL) == 0) 495 send_sig(SIGPIPE, current, 0); 496 ret = -EPIPE; 497 goto out; 498 } 499 500 /* 501 * If it wasn't empty we try to merge new data into 502 * the last buffer. 503 * 504 * That naturally merges small writes, but it also 505 * page-aligns the rest of the writes for large writes 506 * spanning multiple pages. 507 */ 508 head = pipe->head; 509 was_empty = pipe_empty(head, pipe->tail); 510 chars = total_len & (PAGE_SIZE-1); 511 if (chars && !was_empty) { 512 struct pipe_buffer *buf = pipe_buf(pipe, head - 1); 513 int offset = buf->offset + buf->len; 514 515 if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) && 516 offset + chars <= PAGE_SIZE) { 517 ret = pipe_buf_confirm(pipe, buf); 518 if (ret) 519 goto out; 520 521 ret = copy_page_from_iter(buf->page, offset, chars, from); 522 if (unlikely(ret < chars)) { 523 ret = -EFAULT; 524 goto out; 525 } 526 527 buf->len += ret; 528 if (!iov_iter_count(from)) 529 goto out; 530 } 531 } 532 533 for (;;) { 534 if (!pipe->readers) { 535 if ((iocb->ki_flags & IOCB_NOSIGNAL) == 0) 536 send_sig(SIGPIPE, current, 0); 537 if (!ret) 538 ret = -EPIPE; 539 break; 540 } 541 542 head = pipe->head; 543 if (!pipe_full(head, pipe->tail, pipe->max_usage)) { 544 struct pipe_buffer *buf; 545 struct page *page; 546 int copied; 547 548 page = anon_pipe_get_page(pipe); 549 if (unlikely(!page)) { 550 if (!ret) 551 ret = -ENOMEM; 552 break; 553 } 554 555 copied = copy_page_from_iter(page, 0, PAGE_SIZE, from); 556 if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) { 557 anon_pipe_put_page(pipe, page); 558 if (!ret) 559 ret = -EFAULT; 560 break; 561 } 562 563 pipe->head = head + 1; 564 /* Insert it into the buffer array */ 565 buf = pipe_buf(pipe, head); 566 buf->page = page; 567 buf->ops = &anon_pipe_buf_ops; 568 buf->offset = 0; 569 if (is_packetized(filp)) 570 buf->flags = PIPE_BUF_FLAG_PACKET; 571 else 572 buf->flags = PIPE_BUF_FLAG_CAN_MERGE; 573 574 buf->len = copied; 575 ret += copied; 576 577 if (!iov_iter_count(from)) 578 break; 579 580 continue; 581 } 582 583 /* Wait for buffer space to become available. */ 584 if ((filp->f_flags & O_NONBLOCK) || 585 (iocb->ki_flags & IOCB_NOWAIT)) { 586 if (!ret) 587 ret = -EAGAIN; 588 break; 589 } 590 if (signal_pending(current)) { 591 if (!ret) 592 ret = -ERESTARTSYS; 593 break; 594 } 595 596 /* 597 * We're going to release the pipe lock and wait for more 598 * space. We wake up any readers if necessary, and then 599 * after waiting we need to re-check whether the pipe 600 * become empty while we dropped the lock. 601 */ 602 mutex_unlock(&pipe->mutex); 603 if (was_empty) 604 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); 605 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 606 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe)); 607 mutex_lock(&pipe->mutex); 608 was_empty = pipe_is_empty(pipe); 609 wake_next_writer = true; 610 } 611 out: 612 if (pipe_is_full(pipe)) 613 wake_next_writer = false; 614 anon_pipe_trim_and_unlock(pipe); 615 616 /* 617 * If we do do a wakeup event, we do a 'sync' wakeup, because we 618 * want the reader to start processing things asap, rather than 619 * leave the data pending. 620 * 621 * This is particularly important for small writes, because of 622 * how (for example) the GNU make jobserver uses small writes to 623 * wake up pending jobs 624 * 625 * ->pseudo_edgetrigger enables per-write wakeups, see pipe_poll() 626 */ 627 if (was_empty || READ_ONCE(pipe->pseudo_edgetrigger)) 628 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); 629 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 630 if (wake_next_writer) 631 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); 632 return ret; 633 } 634 635 static ssize_t 636 fifo_pipe_write(struct kiocb *iocb, struct iov_iter *from) 637 { 638 int ret = anon_pipe_write(iocb, from); 639 if (ret > 0) { 640 struct file *filp = iocb->ki_filp; 641 if (sb_start_write_trylock(file_inode(filp)->i_sb)) { 642 int err = file_update_time(filp); 643 if (err) 644 ret = err; 645 sb_end_write(file_inode(filp)->i_sb); 646 } 647 } 648 return ret; 649 } 650 651 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 652 { 653 struct pipe_inode_info *pipe = filp->private_data; 654 unsigned int count, head, tail; 655 656 switch (cmd) { 657 case FIONREAD: 658 mutex_lock(&pipe->mutex); 659 count = 0; 660 head = pipe->head; 661 tail = pipe->tail; 662 663 while (!pipe_empty(head, tail)) { 664 count += pipe_buf(pipe, tail)->len; 665 tail++; 666 } 667 mutex_unlock(&pipe->mutex); 668 669 return put_user(count, (int __user *)arg); 670 671 #ifdef CONFIG_WATCH_QUEUE 672 case IOC_WATCH_QUEUE_SET_SIZE: { 673 int ret; 674 mutex_lock(&pipe->mutex); 675 ret = watch_queue_set_size(pipe, arg); 676 mutex_unlock(&pipe->mutex); 677 return ret; 678 } 679 680 case IOC_WATCH_QUEUE_SET_FILTER: 681 return watch_queue_set_filter( 682 pipe, (struct watch_notification_filter __user *)arg); 683 #endif 684 685 default: 686 return -ENOIOCTLCMD; 687 } 688 } 689 690 static __poll_t 691 pipe_poll(struct file *filp, poll_table *wait) 692 { 693 __poll_t mask; 694 struct pipe_inode_info *pipe = filp->private_data; 695 union pipe_index idx; 696 697 /* 698 * Legacy epoll(EPOLLET) users depend on historical per-write wakeups, 699 * see 3a34b13a88ca ("pipe: make pipe writes always wake up readers") 700 * and the ->pseudo_edgetrigger check in anon_pipe_write(). 701 * Currently io_uring sets EPOLLET for multishot polls, so it gets the 702 * same behaviour. 703 */ 704 if ((filp->f_mode & FMODE_READ) && 705 wait && (wait->_key & EPOLLET) && 706 unlikely(!READ_ONCE(pipe->pseudo_edgetrigger))) 707 WRITE_ONCE(pipe->pseudo_edgetrigger, true); 708 709 /* 710 * Reading pipe state only -- no need for acquiring the semaphore. 711 * 712 * But because this is racy, the code has to add the 713 * entry to the poll table _first_ .. 714 */ 715 if (filp->f_mode & FMODE_READ) 716 poll_wait(filp, &pipe->rd_wait, wait); 717 if (filp->f_mode & FMODE_WRITE) 718 poll_wait(filp, &pipe->wr_wait, wait); 719 720 /* 721 * .. and only then can you do the racy tests. That way, 722 * if something changes and you got it wrong, the poll 723 * table entry will wake you up and fix it. 724 */ 725 idx.head_tail = READ_ONCE(pipe->head_tail); 726 727 mask = 0; 728 if (filp->f_mode & FMODE_READ) { 729 if (!pipe_empty(idx.head, idx.tail)) 730 mask |= EPOLLIN | EPOLLRDNORM; 731 if (!pipe->writers && filp->f_pipe != pipe->w_counter) 732 mask |= EPOLLHUP; 733 } 734 735 if (filp->f_mode & FMODE_WRITE) { 736 if (!pipe_full(idx.head, idx.tail, pipe->max_usage)) 737 mask |= EPOLLOUT | EPOLLWRNORM; 738 /* 739 * Most Unices do not set EPOLLERR for FIFOs but on Linux they 740 * behave exactly like pipes for poll(). 741 */ 742 if (!pipe->readers) 743 mask |= EPOLLERR; 744 } 745 746 return mask; 747 } 748 749 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe) 750 { 751 int kill = 0; 752 753 spin_lock(&inode->i_lock); 754 if (!--pipe->files) { 755 inode->i_pipe = NULL; 756 kill = 1; 757 } 758 spin_unlock(&inode->i_lock); 759 760 if (kill) 761 free_pipe_info(pipe); 762 } 763 764 static int 765 pipe_release(struct inode *inode, struct file *file) 766 { 767 struct pipe_inode_info *pipe = file->private_data; 768 769 mutex_lock(&pipe->mutex); 770 if (file->f_mode & FMODE_READ) 771 pipe->readers--; 772 if (file->f_mode & FMODE_WRITE) 773 pipe->writers--; 774 775 /* Was that the last reader or writer, but not the other side? */ 776 if (!pipe->readers != !pipe->writers) { 777 wake_up_interruptible_all(&pipe->rd_wait); 778 wake_up_interruptible_all(&pipe->wr_wait); 779 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 780 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 781 } 782 mutex_unlock(&pipe->mutex); 783 784 put_pipe_info(inode, pipe); 785 return 0; 786 } 787 788 static int 789 pipe_fasync(int fd, struct file *filp, int on) 790 { 791 struct pipe_inode_info *pipe = filp->private_data; 792 int retval = 0; 793 794 mutex_lock(&pipe->mutex); 795 if (filp->f_mode & FMODE_READ) 796 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); 797 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) { 798 int read_ret = retval; 799 800 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); 801 /* Undo only a reader entry added by this call. */ 802 if (retval < 0 && read_ret > 0) 803 fasync_helper(-1, filp, 0, &pipe->fasync_readers); 804 } 805 mutex_unlock(&pipe->mutex); 806 return retval; 807 } 808 809 unsigned long account_pipe_buffers(struct user_struct *user, 810 unsigned long old, unsigned long new) 811 { 812 return atomic_long_add_return(new - old, &user->pipe_bufs); 813 } 814 815 bool too_many_pipe_buffers_soft(unsigned long user_bufs) 816 { 817 unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft); 818 819 return soft_limit && user_bufs > soft_limit; 820 } 821 822 bool too_many_pipe_buffers_hard(unsigned long user_bufs) 823 { 824 unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard); 825 826 return hard_limit && user_bufs > hard_limit; 827 } 828 829 bool pipe_is_unprivileged_user(void) 830 { 831 return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN); 832 } 833 834 struct pipe_inode_info *alloc_pipe_info(void) 835 { 836 struct pipe_inode_info *pipe; 837 unsigned long pipe_bufs = PIPE_DEF_BUFFERS; 838 struct user_struct *user = get_current_user(); 839 unsigned long user_bufs; 840 unsigned int max_size = READ_ONCE(pipe_max_size); 841 842 pipe = kzalloc_obj(struct pipe_inode_info, GFP_KERNEL_ACCOUNT); 843 if (pipe == NULL) 844 goto out_free_uid; 845 846 if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE)) 847 pipe_bufs = max_size >> PAGE_SHIFT; 848 849 user_bufs = account_pipe_buffers(user, 0, pipe_bufs); 850 851 if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) { 852 user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS); 853 pipe_bufs = PIPE_MIN_DEF_BUFFERS; 854 } 855 856 if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user()) 857 goto out_revert_acct; 858 859 pipe->bufs = kzalloc_objs(struct pipe_buffer, pipe_bufs, 860 GFP_KERNEL_ACCOUNT); 861 862 if (pipe->bufs) { 863 init_waitqueue_head(&pipe->rd_wait); 864 init_waitqueue_head(&pipe->wr_wait); 865 pipe->r_counter = pipe->w_counter = 1; 866 pipe->max_usage = pipe_bufs; 867 pipe->ring_size = pipe_bufs; 868 pipe->nr_accounted = pipe_bufs; 869 pipe->user = user; 870 mutex_init(&pipe->mutex); 871 lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL); 872 return pipe; 873 } 874 875 out_revert_acct: 876 (void) account_pipe_buffers(user, pipe_bufs, 0); 877 kfree(pipe); 878 out_free_uid: 879 free_uid(user); 880 return NULL; 881 } 882 883 void free_pipe_info(struct pipe_inode_info *pipe) 884 { 885 unsigned int i; 886 887 #ifdef CONFIG_WATCH_QUEUE 888 if (pipe->watch_queue) 889 watch_queue_clear(pipe->watch_queue); 890 #endif 891 892 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); 893 free_uid(pipe->user); 894 for (i = 0; i < pipe->ring_size; i++) { 895 struct pipe_buffer *buf = pipe->bufs + i; 896 if (buf->ops) 897 pipe_buf_release(pipe, buf); 898 } 899 #ifdef CONFIG_WATCH_QUEUE 900 if (pipe->watch_queue) 901 put_watch_queue(pipe->watch_queue); 902 #endif 903 for (i = 0; i < pipe->prealloc.count; i++) 904 __free_page(pipe->prealloc.pages[i]); 905 kfree(pipe->bufs); 906 kfree(pipe); 907 } 908 909 static struct vfsmount *pipe_mnt __ro_after_init; 910 911 /* 912 * pipefs_dname() is called from d_path(). 913 */ 914 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen) 915 { 916 return dynamic_dname(buffer, buflen, "pipe:[%llu]", 917 d_inode(dentry)->i_ino); 918 } 919 920 static const struct dentry_operations pipefs_dentry_operations = { 921 .d_dname = pipefs_dname, 922 }; 923 924 static const struct file_operations pipeanon_fops; 925 926 static struct inode * get_pipe_inode(void) 927 { 928 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb); 929 struct pipe_inode_info *pipe; 930 931 if (!inode) 932 goto fail_inode; 933 934 inode->i_ino = get_next_ino(); 935 936 pipe = alloc_pipe_info(); 937 if (!pipe) 938 goto fail_iput; 939 940 inode->i_pipe = pipe; 941 pipe->files = 2; 942 pipe->readers = pipe->writers = 1; 943 inode->i_fop = &pipeanon_fops; 944 945 /* 946 * Mark the inode dirty from the very beginning, 947 * that way it will never be moved to the dirty 948 * list because "mark_inode_dirty()" will think 949 * that it already _is_ on the dirty list. 950 */ 951 inode_state_assign_raw(inode, I_DIRTY); 952 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR; 953 inode->i_uid = current_fsuid(); 954 inode->i_gid = current_fsgid(); 955 simple_inode_init_ts(inode); 956 957 return inode; 958 959 fail_iput: 960 iput(inode); 961 962 fail_inode: 963 return NULL; 964 } 965 966 int create_pipe_files(struct file **res, int flags) 967 { 968 struct inode *inode = get_pipe_inode(); 969 struct file *f; 970 int error; 971 972 if (!inode) 973 return -ENFILE; 974 975 if (flags & O_NOTIFICATION_PIPE) { 976 error = watch_queue_init(inode->i_pipe); 977 if (error) { 978 free_pipe_info(inode->i_pipe); 979 iput(inode); 980 return error; 981 } 982 } 983 984 f = alloc_file_pseudo(inode, pipe_mnt, "", 985 O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)), 986 &pipeanon_fops); 987 if (IS_ERR(f)) { 988 free_pipe_info(inode->i_pipe); 989 iput(inode); 990 return PTR_ERR(f); 991 } 992 993 f->private_data = inode->i_pipe; 994 f->f_pipe = 0; 995 996 res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK), 997 &pipeanon_fops); 998 if (IS_ERR(res[0])) { 999 put_pipe_info(inode, inode->i_pipe); 1000 fput(f); 1001 return PTR_ERR(res[0]); 1002 } 1003 res[0]->private_data = inode->i_pipe; 1004 res[0]->f_pipe = 0; 1005 res[1] = f; 1006 stream_open(inode, res[0]); 1007 stream_open(inode, res[1]); 1008 1009 /* pipe groks IOCB_NOWAIT */ 1010 res[0]->f_mode |= FMODE_NOWAIT; 1011 res[1]->f_mode |= FMODE_NOWAIT; 1012 1013 /* 1014 * Disable permission and pre-content events, but enable legacy 1015 * inotify events for legacy users. 1016 */ 1017 file_set_fsnotify_mode(res[0], FMODE_NONOTIFY_PERM); 1018 file_set_fsnotify_mode(res[1], FMODE_NONOTIFY_PERM); 1019 return 0; 1020 } 1021 1022 static int __do_pipe_flags(int *fd, struct file **files, int flags) 1023 { 1024 int error; 1025 int fdw, fdr; 1026 1027 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE)) 1028 return -EINVAL; 1029 1030 error = create_pipe_files(files, flags); 1031 if (error) 1032 return error; 1033 1034 error = get_unused_fd_flags(flags); 1035 if (error < 0) 1036 goto err_read_pipe; 1037 fdr = error; 1038 1039 error = get_unused_fd_flags(flags); 1040 if (error < 0) 1041 goto err_fdr; 1042 fdw = error; 1043 1044 audit_fd_pair(fdr, fdw); 1045 fd[0] = fdr; 1046 fd[1] = fdw; 1047 return 0; 1048 1049 err_fdr: 1050 put_unused_fd(fdr); 1051 err_read_pipe: 1052 fput(files[0]); 1053 fput(files[1]); 1054 return error; 1055 } 1056 1057 int do_pipe_flags(int *fd, int flags) 1058 { 1059 struct file *files[2]; 1060 int error = __do_pipe_flags(fd, files, flags); 1061 if (!error) { 1062 fd_install(fd[0], files[0]); 1063 fd_install(fd[1], files[1]); 1064 } 1065 return error; 1066 } 1067 1068 /* 1069 * sys_pipe() is the normal C calling standard for creating 1070 * a pipe. It's not the way Unix traditionally does this, though. 1071 */ 1072 static int do_pipe2(int __user *fildes, int flags) 1073 { 1074 struct file *files[2]; 1075 int fd[2]; 1076 int error; 1077 1078 error = __do_pipe_flags(fd, files, flags); 1079 if (!error) { 1080 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) { 1081 fput(files[0]); 1082 fput(files[1]); 1083 put_unused_fd(fd[0]); 1084 put_unused_fd(fd[1]); 1085 error = -EFAULT; 1086 } else { 1087 fd_install(fd[0], files[0]); 1088 fd_install(fd[1], files[1]); 1089 } 1090 } 1091 return error; 1092 } 1093 1094 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags) 1095 { 1096 return do_pipe2(fildes, flags); 1097 } 1098 1099 SYSCALL_DEFINE1(pipe, int __user *, fildes) 1100 { 1101 return do_pipe2(fildes, 0); 1102 } 1103 1104 /* 1105 * This is the stupid "wait for pipe to be readable or writable" 1106 * model. 1107 * 1108 * See pipe_read/write() for the proper kind of exclusive wait, 1109 * but that requires that we wake up any other readers/writers 1110 * if we then do not end up reading everything (ie the whole 1111 * "wake_next_reader/writer" logic in pipe_read/write()). 1112 */ 1113 void pipe_wait_readable(struct pipe_inode_info *pipe) 1114 { 1115 pipe_unlock(pipe); 1116 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe)); 1117 pipe_lock(pipe); 1118 } 1119 1120 void pipe_wait_writable(struct pipe_inode_info *pipe) 1121 { 1122 pipe_unlock(pipe); 1123 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe)); 1124 pipe_lock(pipe); 1125 } 1126 1127 /* 1128 * This depends on both the wait (here) and the wakeup (wake_up_partner) 1129 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot 1130 * race with the count check and waitqueue prep. 1131 * 1132 * Normally in order to avoid races, you'd do the prepare_to_wait() first, 1133 * then check the condition you're waiting for, and only then sleep. But 1134 * because of the pipe lock, we can check the condition before being on 1135 * the wait queue. 1136 * 1137 * We use the 'rd_wait' waitqueue for pipe partner waiting. 1138 */ 1139 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) 1140 { 1141 DEFINE_WAIT(rdwait); 1142 int cur = *cnt; 1143 1144 while (cur == *cnt) { 1145 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE); 1146 pipe_unlock(pipe); 1147 schedule(); 1148 finish_wait(&pipe->rd_wait, &rdwait); 1149 pipe_lock(pipe); 1150 if (signal_pending(current)) 1151 break; 1152 } 1153 return cur == *cnt ? -ERESTARTSYS : 0; 1154 } 1155 1156 static void wake_up_partner(struct pipe_inode_info *pipe) 1157 { 1158 wake_up_interruptible_all(&pipe->rd_wait); 1159 } 1160 1161 static int fifo_open(struct inode *inode, struct file *filp) 1162 { 1163 bool is_pipe = inode->i_fop == &pipeanon_fops; 1164 struct pipe_inode_info *pipe; 1165 int ret; 1166 1167 filp->f_pipe = 0; 1168 1169 spin_lock(&inode->i_lock); 1170 if (inode->i_pipe) { 1171 pipe = inode->i_pipe; 1172 pipe->files++; 1173 spin_unlock(&inode->i_lock); 1174 } else { 1175 spin_unlock(&inode->i_lock); 1176 pipe = alloc_pipe_info(); 1177 if (!pipe) 1178 return -ENOMEM; 1179 pipe->files = 1; 1180 spin_lock(&inode->i_lock); 1181 if (unlikely(inode->i_pipe)) { 1182 inode->i_pipe->files++; 1183 spin_unlock(&inode->i_lock); 1184 free_pipe_info(pipe); 1185 pipe = inode->i_pipe; 1186 } else { 1187 inode->i_pipe = pipe; 1188 spin_unlock(&inode->i_lock); 1189 } 1190 } 1191 filp->private_data = pipe; 1192 /* OK, we have a pipe and it's pinned down */ 1193 1194 mutex_lock(&pipe->mutex); 1195 1196 /* We can only do regular read/write on fifos */ 1197 stream_open(inode, filp); 1198 1199 switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) { 1200 case FMODE_READ: 1201 /* 1202 * O_RDONLY 1203 * POSIX.1 says that O_NONBLOCK means return with the FIFO 1204 * opened, even when there is no process writing the FIFO. 1205 */ 1206 pipe->r_counter++; 1207 if (pipe->readers++ == 0) 1208 wake_up_partner(pipe); 1209 1210 if (!is_pipe && !pipe->writers) { 1211 if ((filp->f_flags & O_NONBLOCK)) { 1212 /* suppress EPOLLHUP until we have 1213 * seen a writer */ 1214 filp->f_pipe = pipe->w_counter; 1215 } else { 1216 if (wait_for_partner(pipe, &pipe->w_counter)) 1217 goto err_rd; 1218 } 1219 } 1220 break; 1221 1222 case FMODE_WRITE: 1223 /* 1224 * O_WRONLY 1225 * POSIX.1 says that O_NONBLOCK means return -1 with 1226 * errno=ENXIO when there is no process reading the FIFO. 1227 */ 1228 ret = -ENXIO; 1229 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) 1230 goto err; 1231 1232 pipe->w_counter++; 1233 if (!pipe->writers++) 1234 wake_up_partner(pipe); 1235 1236 if (!is_pipe && !pipe->readers) { 1237 if (wait_for_partner(pipe, &pipe->r_counter)) 1238 goto err_wr; 1239 } 1240 break; 1241 1242 case FMODE_READ | FMODE_WRITE: 1243 /* 1244 * O_RDWR 1245 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set. 1246 * This implementation will NEVER block on a O_RDWR open, since 1247 * the process can at least talk to itself. 1248 */ 1249 1250 pipe->readers++; 1251 pipe->writers++; 1252 pipe->r_counter++; 1253 pipe->w_counter++; 1254 if (pipe->readers == 1 || pipe->writers == 1) 1255 wake_up_partner(pipe); 1256 break; 1257 1258 default: 1259 ret = -EINVAL; 1260 goto err; 1261 } 1262 1263 /* Ok! */ 1264 mutex_unlock(&pipe->mutex); 1265 return 0; 1266 1267 err_rd: 1268 if (!--pipe->readers) 1269 wake_up_interruptible(&pipe->wr_wait); 1270 ret = -ERESTARTSYS; 1271 goto err; 1272 1273 err_wr: 1274 if (!--pipe->writers) 1275 wake_up_interruptible_all(&pipe->rd_wait); 1276 ret = -ERESTARTSYS; 1277 goto err; 1278 1279 err: 1280 mutex_unlock(&pipe->mutex); 1281 1282 put_pipe_info(inode, pipe); 1283 return ret; 1284 } 1285 1286 const struct file_operations pipefifo_fops = { 1287 .open = fifo_open, 1288 .read_iter = fifo_pipe_read, 1289 .write_iter = fifo_pipe_write, 1290 .poll = pipe_poll, 1291 .unlocked_ioctl = pipe_ioctl, 1292 .release = pipe_release, 1293 .fasync = pipe_fasync, 1294 .splice_write = iter_file_splice_write, 1295 }; 1296 1297 static const struct file_operations pipeanon_fops = { 1298 .open = fifo_open, 1299 .read_iter = anon_pipe_read, 1300 .write_iter = anon_pipe_write, 1301 .poll = pipe_poll, 1302 .unlocked_ioctl = pipe_ioctl, 1303 .release = pipe_release, 1304 .fasync = pipe_fasync, 1305 .splice_write = iter_file_splice_write, 1306 }; 1307 1308 /* 1309 * Currently we rely on the pipe array holding a power-of-2 number 1310 * of pages. Returns 0 on error. 1311 */ 1312 unsigned int round_pipe_size(unsigned int size) 1313 { 1314 if (size > (1U << 31)) 1315 return 0; 1316 1317 /* Minimum pipe size, as required by POSIX */ 1318 if (size < PAGE_SIZE) 1319 return PAGE_SIZE; 1320 1321 return roundup_pow_of_two(size); 1322 } 1323 1324 /* 1325 * Resize the pipe ring to a number of slots. 1326 * 1327 * Note the pipe can be reduced in capacity, but only if the current 1328 * occupancy doesn't exceed nr_slots; if it does, EBUSY will be 1329 * returned instead. 1330 */ 1331 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) 1332 { 1333 struct pipe_buffer *bufs; 1334 unsigned int head, tail, mask, n; 1335 1336 /* nr_slots larger than limits of pipe->{head,tail} */ 1337 if (unlikely(nr_slots > (pipe_index_t)-1u)) 1338 return -EINVAL; 1339 1340 bufs = kzalloc_objs(*bufs, nr_slots, GFP_KERNEL_ACCOUNT | __GFP_NOWARN); 1341 if (unlikely(!bufs)) 1342 return -ENOMEM; 1343 1344 spin_lock_irq(&pipe->rd_wait.lock); 1345 mask = pipe->ring_size - 1; 1346 head = pipe->head; 1347 tail = pipe->tail; 1348 1349 n = pipe_occupancy(head, tail); 1350 if (nr_slots < n) { 1351 spin_unlock_irq(&pipe->rd_wait.lock); 1352 kfree(bufs); 1353 return -EBUSY; 1354 } 1355 1356 /* 1357 * The pipe array wraps around, so just start the new one at zero 1358 * and adjust the indices. 1359 */ 1360 if (n > 0) { 1361 unsigned int h = head & mask; 1362 unsigned int t = tail & mask; 1363 if (h > t) { 1364 memcpy(bufs, pipe->bufs + t, 1365 n * sizeof(struct pipe_buffer)); 1366 } else { 1367 unsigned int tsize = pipe->ring_size - t; 1368 if (h > 0) 1369 memcpy(bufs + tsize, pipe->bufs, 1370 h * sizeof(struct pipe_buffer)); 1371 memcpy(bufs, pipe->bufs + t, 1372 tsize * sizeof(struct pipe_buffer)); 1373 } 1374 } 1375 1376 head = n; 1377 tail = 0; 1378 1379 kfree(pipe->bufs); 1380 pipe->bufs = bufs; 1381 pipe->ring_size = nr_slots; 1382 if (pipe->max_usage > nr_slots) 1383 pipe->max_usage = nr_slots; 1384 pipe->tail = tail; 1385 pipe->head = head; 1386 1387 if (!pipe_has_watch_queue(pipe)) { 1388 pipe->max_usage = nr_slots; 1389 pipe->nr_accounted = nr_slots; 1390 } 1391 1392 spin_unlock_irq(&pipe->rd_wait.lock); 1393 1394 /* This might have made more room for writers */ 1395 wake_up_interruptible_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); 1396 return 0; 1397 } 1398 1399 /* 1400 * Allocate a new array of pipe buffers and copy the info over. Returns the 1401 * pipe size if successful, or return -ERROR on error. 1402 */ 1403 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg) 1404 { 1405 unsigned long user_bufs; 1406 unsigned int nr_slots, size; 1407 long ret = 0; 1408 1409 if (pipe_has_watch_queue(pipe)) 1410 return -EBUSY; 1411 1412 size = round_pipe_size(arg); 1413 nr_slots = size >> PAGE_SHIFT; 1414 1415 if (!nr_slots) 1416 return -EINVAL; 1417 1418 /* 1419 * If trying to increase the pipe capacity, check that an 1420 * unprivileged user is not trying to exceed various limits 1421 * (soft limit check here, hard limit check just below). 1422 * Decreasing the pipe capacity is always permitted, even 1423 * if the user is currently over a limit. 1424 */ 1425 if (nr_slots > pipe->max_usage && 1426 size > pipe_max_size && !capable(CAP_SYS_RESOURCE)) 1427 return -EPERM; 1428 1429 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); 1430 1431 if (nr_slots > pipe->max_usage && 1432 (too_many_pipe_buffers_hard(user_bufs) || 1433 too_many_pipe_buffers_soft(user_bufs)) && 1434 pipe_is_unprivileged_user()) { 1435 ret = -EPERM; 1436 goto out_revert_acct; 1437 } 1438 1439 ret = pipe_resize_ring(pipe, nr_slots); 1440 if (ret < 0) 1441 goto out_revert_acct; 1442 1443 return pipe->max_usage * PAGE_SIZE; 1444 1445 out_revert_acct: 1446 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); 1447 return ret; 1448 } 1449 1450 /* 1451 * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is 1452 * not enough to verify that this is a pipe. 1453 */ 1454 struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice) 1455 { 1456 struct pipe_inode_info *pipe = file->private_data; 1457 1458 if (!pipe) 1459 return NULL; 1460 if (file->f_op != &pipefifo_fops && file->f_op != &pipeanon_fops) 1461 return NULL; 1462 if (for_splice && pipe_has_watch_queue(pipe)) 1463 return NULL; 1464 return pipe; 1465 } 1466 1467 long pipe_fcntl(struct file *file, unsigned int cmd, unsigned int arg) 1468 { 1469 struct pipe_inode_info *pipe; 1470 long ret; 1471 1472 pipe = get_pipe_info(file, false); 1473 if (!pipe) 1474 return -EBADF; 1475 1476 mutex_lock(&pipe->mutex); 1477 1478 switch (cmd) { 1479 case F_SETPIPE_SZ: 1480 ret = pipe_set_size(pipe, arg); 1481 break; 1482 case F_GETPIPE_SZ: 1483 ret = pipe->max_usage * PAGE_SIZE; 1484 break; 1485 default: 1486 ret = -EINVAL; 1487 break; 1488 } 1489 1490 mutex_unlock(&pipe->mutex); 1491 return ret; 1492 } 1493 1494 static const struct super_operations pipefs_ops = { 1495 .destroy_inode = free_inode_nonrcu, 1496 .statfs = simple_statfs, 1497 }; 1498 1499 /* 1500 * pipefs should _never_ be mounted by userland - too much of security hassle, 1501 * no real gain from having the whole file system mounted. So we don't need 1502 * any operations on the root directory. However, we need a non-trivial 1503 * d_name - pipe: will go nicely and kill the special-casing in procfs. 1504 */ 1505 1506 static int pipefs_init_fs_context(struct fs_context *fc) 1507 { 1508 struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC); 1509 if (!ctx) 1510 return -ENOMEM; 1511 ctx->ops = &pipefs_ops; 1512 ctx->dops = &pipefs_dentry_operations; 1513 return 0; 1514 } 1515 1516 static struct file_system_type pipe_fs_type = { 1517 .name = "pipefs", 1518 .init_fs_context = pipefs_init_fs_context, 1519 .kill_sb = kill_anon_super, 1520 }; 1521 1522 #ifdef CONFIG_SYSCTL 1523 1524 static ulong round_pipe_size_ul(ulong size) 1525 { 1526 return round_pipe_size(size); 1527 } 1528 1529 static int u2k_pipe_maxsz(const ulong *u_ptr, uint *k_ptr) 1530 { 1531 return proc_uint_u2k_conv_uop(u_ptr, k_ptr, round_pipe_size_ul); 1532 } 1533 1534 static int do_proc_uint_conv_pipe_maxsz(bool *negp, ulong *u_ptr, uint *k_ptr, 1535 int dir, const struct ctl_table *table) 1536 { 1537 return proc_uint_conv(u_ptr, k_ptr, dir, table, true, 1538 u2k_pipe_maxsz, 1539 proc_uint_k2u_conv); 1540 } 1541 1542 static int proc_dopipe_max_size(const struct ctl_table *table, int write, 1543 void *buffer, size_t *lenp, loff_t *ppos) 1544 { 1545 return proc_douintvec_conv(table, write, buffer, lenp, ppos, 1546 do_proc_uint_conv_pipe_maxsz); 1547 } 1548 1549 static const struct ctl_table fs_pipe_sysctls[] = { 1550 { 1551 .procname = "pipe-max-size", 1552 .data = &pipe_max_size, 1553 .maxlen = sizeof(pipe_max_size), 1554 .mode = 0644, 1555 .proc_handler = proc_dopipe_max_size, 1556 .extra1 = SYSCTL_ONE, 1557 }, 1558 { 1559 .procname = "pipe-user-pages-hard", 1560 .data = &pipe_user_pages_hard, 1561 .maxlen = sizeof(pipe_user_pages_hard), 1562 .mode = 0644, 1563 .proc_handler = proc_doulongvec_minmax, 1564 }, 1565 { 1566 .procname = "pipe-user-pages-soft", 1567 .data = &pipe_user_pages_soft, 1568 .maxlen = sizeof(pipe_user_pages_soft), 1569 .mode = 0644, 1570 .proc_handler = proc_doulongvec_minmax, 1571 }, 1572 }; 1573 #endif 1574 1575 static int __init init_pipe_fs(void) 1576 { 1577 int err = register_filesystem(&pipe_fs_type); 1578 1579 if (!err) { 1580 pipe_mnt = kern_mount(&pipe_fs_type); 1581 if (IS_ERR(pipe_mnt)) { 1582 err = PTR_ERR(pipe_mnt); 1583 unregister_filesystem(&pipe_fs_type); 1584 } 1585 } 1586 #ifdef CONFIG_SYSCTL 1587 register_sysctl_init("fs", fs_pipe_sysctls); 1588 #endif 1589 return err; 1590 } 1591 1592 fs_initcall(init_pipe_fs); 1593