1 /* 2 * linux/fs/pipe.c 3 * 4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds 5 */ 6 7 #include <linux/mm.h> 8 #include <linux/file.h> 9 #include <linux/poll.h> 10 #include <linux/slab.h> 11 #include <linux/module.h> 12 #include <linux/init.h> 13 #include <linux/fs.h> 14 #include <linux/log2.h> 15 #include <linux/mount.h> 16 #include <linux/magic.h> 17 #include <linux/pipe_fs_i.h> 18 #include <linux/uio.h> 19 #include <linux/highmem.h> 20 #include <linux/pagemap.h> 21 #include <linux/audit.h> 22 #include <linux/syscalls.h> 23 #include <linux/fcntl.h> 24 25 #include <asm/uaccess.h> 26 #include <asm/ioctls.h> 27 28 #include "internal.h" 29 30 /* 31 * The max size that a non-root user is allowed to grow the pipe. Can 32 * be set by root in /proc/sys/fs/pipe-max-size 33 */ 34 unsigned int pipe_max_size = 1048576; 35 36 /* 37 * Minimum pipe size, as required by POSIX 38 */ 39 unsigned int pipe_min_size = PAGE_SIZE; 40 41 /* 42 * We use a start+len construction, which provides full use of the 43 * allocated memory. 44 * -- Florian Coosmann (FGC) 45 * 46 * Reads with count = 0 should always return 0. 47 * -- Julian Bradfield 1999-06-07. 48 * 49 * FIFOs and Pipes now generate SIGIO for both readers and writers. 50 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 51 * 52 * pipe_read & write cleanup 53 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 54 */ 55 56 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass) 57 { 58 if (pipe->files) 59 mutex_lock_nested(&pipe->mutex, subclass); 60 } 61 62 void pipe_lock(struct pipe_inode_info *pipe) 63 { 64 /* 65 * pipe_lock() nests non-pipe inode locks (for writing to a file) 66 */ 67 pipe_lock_nested(pipe, I_MUTEX_PARENT); 68 } 69 EXPORT_SYMBOL(pipe_lock); 70 71 void pipe_unlock(struct pipe_inode_info *pipe) 72 { 73 if (pipe->files) 74 mutex_unlock(&pipe->mutex); 75 } 76 EXPORT_SYMBOL(pipe_unlock); 77 78 static inline void __pipe_lock(struct pipe_inode_info *pipe) 79 { 80 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT); 81 } 82 83 static inline void __pipe_unlock(struct pipe_inode_info *pipe) 84 { 85 mutex_unlock(&pipe->mutex); 86 } 87 88 void pipe_double_lock(struct pipe_inode_info *pipe1, 89 struct pipe_inode_info *pipe2) 90 { 91 BUG_ON(pipe1 == pipe2); 92 93 if (pipe1 < pipe2) { 94 pipe_lock_nested(pipe1, I_MUTEX_PARENT); 95 pipe_lock_nested(pipe2, I_MUTEX_CHILD); 96 } else { 97 pipe_lock_nested(pipe2, I_MUTEX_PARENT); 98 pipe_lock_nested(pipe1, I_MUTEX_CHILD); 99 } 100 } 101 102 /* Drop the inode semaphore and wait for a pipe event, atomically */ 103 void pipe_wait(struct pipe_inode_info *pipe) 104 { 105 DEFINE_WAIT(wait); 106 107 /* 108 * Pipes are system-local resources, so sleeping on them 109 * is considered a noninteractive wait: 110 */ 111 prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE); 112 pipe_unlock(pipe); 113 schedule(); 114 finish_wait(&pipe->wait, &wait); 115 pipe_lock(pipe); 116 } 117 118 static int 119 pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, 120 int atomic) 121 { 122 unsigned long copy; 123 124 while (len > 0) { 125 while (!iov->iov_len) 126 iov++; 127 copy = min_t(unsigned long, len, iov->iov_len); 128 129 if (atomic) { 130 if (__copy_from_user_inatomic(to, iov->iov_base, copy)) 131 return -EFAULT; 132 } else { 133 if (copy_from_user(to, iov->iov_base, copy)) 134 return -EFAULT; 135 } 136 to += copy; 137 len -= copy; 138 iov->iov_base += copy; 139 iov->iov_len -= copy; 140 } 141 return 0; 142 } 143 144 static int 145 pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len, 146 int atomic) 147 { 148 unsigned long copy; 149 150 while (len > 0) { 151 while (!iov->iov_len) 152 iov++; 153 copy = min_t(unsigned long, len, iov->iov_len); 154 155 if (atomic) { 156 if (__copy_to_user_inatomic(iov->iov_base, from, copy)) 157 return -EFAULT; 158 } else { 159 if (copy_to_user(iov->iov_base, from, copy)) 160 return -EFAULT; 161 } 162 from += copy; 163 len -= copy; 164 iov->iov_base += copy; 165 iov->iov_len -= copy; 166 } 167 return 0; 168 } 169 170 /* 171 * Attempt to pre-fault in the user memory, so we can use atomic copies. 172 * Returns the number of bytes not faulted in. 173 */ 174 static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len) 175 { 176 while (!iov->iov_len) 177 iov++; 178 179 while (len > 0) { 180 unsigned long this_len; 181 182 this_len = min_t(unsigned long, len, iov->iov_len); 183 if (fault_in_pages_writeable(iov->iov_base, this_len)) 184 break; 185 186 len -= this_len; 187 iov++; 188 } 189 190 return len; 191 } 192 193 /* 194 * Pre-fault in the user memory, so we can use atomic copies. 195 */ 196 static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len) 197 { 198 while (!iov->iov_len) 199 iov++; 200 201 while (len > 0) { 202 unsigned long this_len; 203 204 this_len = min_t(unsigned long, len, iov->iov_len); 205 fault_in_pages_readable(iov->iov_base, this_len); 206 len -= this_len; 207 iov++; 208 } 209 } 210 211 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, 212 struct pipe_buffer *buf) 213 { 214 struct page *page = buf->page; 215 216 /* 217 * If nobody else uses this page, and we don't already have a 218 * temporary page, let's keep track of it as a one-deep 219 * allocation cache. (Otherwise just release our reference to it) 220 */ 221 if (page_count(page) == 1 && !pipe->tmp_page) 222 pipe->tmp_page = page; 223 else 224 page_cache_release(page); 225 } 226 227 /** 228 * generic_pipe_buf_map - virtually map a pipe buffer 229 * @pipe: the pipe that the buffer belongs to 230 * @buf: the buffer that should be mapped 231 * @atomic: whether to use an atomic map 232 * 233 * Description: 234 * This function returns a kernel virtual address mapping for the 235 * pipe_buffer passed in @buf. If @atomic is set, an atomic map is provided 236 * and the caller has to be careful not to fault before calling 237 * the unmap function. 238 * 239 * Note that this function calls kmap_atomic() if @atomic != 0. 240 */ 241 void *generic_pipe_buf_map(struct pipe_inode_info *pipe, 242 struct pipe_buffer *buf, int atomic) 243 { 244 if (atomic) { 245 buf->flags |= PIPE_BUF_FLAG_ATOMIC; 246 return kmap_atomic(buf->page); 247 } 248 249 return kmap(buf->page); 250 } 251 EXPORT_SYMBOL(generic_pipe_buf_map); 252 253 /** 254 * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer 255 * @pipe: the pipe that the buffer belongs to 256 * @buf: the buffer that should be unmapped 257 * @map_data: the data that the mapping function returned 258 * 259 * Description: 260 * This function undoes the mapping that ->map() provided. 261 */ 262 void generic_pipe_buf_unmap(struct pipe_inode_info *pipe, 263 struct pipe_buffer *buf, void *map_data) 264 { 265 if (buf->flags & PIPE_BUF_FLAG_ATOMIC) { 266 buf->flags &= ~PIPE_BUF_FLAG_ATOMIC; 267 kunmap_atomic(map_data); 268 } else 269 kunmap(buf->page); 270 } 271 EXPORT_SYMBOL(generic_pipe_buf_unmap); 272 273 /** 274 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer 275 * @pipe: the pipe that the buffer belongs to 276 * @buf: the buffer to attempt to steal 277 * 278 * Description: 279 * This function attempts to steal the &struct page attached to 280 * @buf. If successful, this function returns 0 and returns with 281 * the page locked. The caller may then reuse the page for whatever 282 * he wishes; the typical use is insertion into a different file 283 * page cache. 284 */ 285 int generic_pipe_buf_steal(struct pipe_inode_info *pipe, 286 struct pipe_buffer *buf) 287 { 288 struct page *page = buf->page; 289 290 /* 291 * A reference of one is golden, that means that the owner of this 292 * page is the only one holding a reference to it. lock the page 293 * and return OK. 294 */ 295 if (page_count(page) == 1) { 296 lock_page(page); 297 return 0; 298 } 299 300 return 1; 301 } 302 EXPORT_SYMBOL(generic_pipe_buf_steal); 303 304 /** 305 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer 306 * @pipe: the pipe that the buffer belongs to 307 * @buf: the buffer to get a reference to 308 * 309 * Description: 310 * This function grabs an extra reference to @buf. It's used in 311 * in the tee() system call, when we duplicate the buffers in one 312 * pipe into another. 313 */ 314 void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 315 { 316 page_cache_get(buf->page); 317 } 318 EXPORT_SYMBOL(generic_pipe_buf_get); 319 320 /** 321 * generic_pipe_buf_confirm - verify contents of the pipe buffer 322 * @info: the pipe that the buffer belongs to 323 * @buf: the buffer to confirm 324 * 325 * Description: 326 * This function does nothing, because the generic pipe code uses 327 * pages that are always good when inserted into the pipe. 328 */ 329 int generic_pipe_buf_confirm(struct pipe_inode_info *info, 330 struct pipe_buffer *buf) 331 { 332 return 0; 333 } 334 EXPORT_SYMBOL(generic_pipe_buf_confirm); 335 336 /** 337 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer 338 * @pipe: the pipe that the buffer belongs to 339 * @buf: the buffer to put a reference to 340 * 341 * Description: 342 * This function releases a reference to @buf. 343 */ 344 void generic_pipe_buf_release(struct pipe_inode_info *pipe, 345 struct pipe_buffer *buf) 346 { 347 page_cache_release(buf->page); 348 } 349 EXPORT_SYMBOL(generic_pipe_buf_release); 350 351 static const struct pipe_buf_operations anon_pipe_buf_ops = { 352 .can_merge = 1, 353 .map = generic_pipe_buf_map, 354 .unmap = generic_pipe_buf_unmap, 355 .confirm = generic_pipe_buf_confirm, 356 .release = anon_pipe_buf_release, 357 .steal = generic_pipe_buf_steal, 358 .get = generic_pipe_buf_get, 359 }; 360 361 static const struct pipe_buf_operations packet_pipe_buf_ops = { 362 .can_merge = 0, 363 .map = generic_pipe_buf_map, 364 .unmap = generic_pipe_buf_unmap, 365 .confirm = generic_pipe_buf_confirm, 366 .release = anon_pipe_buf_release, 367 .steal = generic_pipe_buf_steal, 368 .get = generic_pipe_buf_get, 369 }; 370 371 static ssize_t 372 pipe_read(struct kiocb *iocb, const struct iovec *_iov, 373 unsigned long nr_segs, loff_t pos) 374 { 375 struct file *filp = iocb->ki_filp; 376 struct pipe_inode_info *pipe = filp->private_data; 377 int do_wakeup; 378 ssize_t ret; 379 struct iovec *iov = (struct iovec *)_iov; 380 size_t total_len; 381 382 total_len = iov_length(iov, nr_segs); 383 /* Null read succeeds. */ 384 if (unlikely(total_len == 0)) 385 return 0; 386 387 do_wakeup = 0; 388 ret = 0; 389 __pipe_lock(pipe); 390 for (;;) { 391 int bufs = pipe->nrbufs; 392 if (bufs) { 393 int curbuf = pipe->curbuf; 394 struct pipe_buffer *buf = pipe->bufs + curbuf; 395 const struct pipe_buf_operations *ops = buf->ops; 396 void *addr; 397 size_t chars = buf->len; 398 int error, atomic; 399 400 if (chars > total_len) 401 chars = total_len; 402 403 error = ops->confirm(pipe, buf); 404 if (error) { 405 if (!ret) 406 ret = error; 407 break; 408 } 409 410 atomic = !iov_fault_in_pages_write(iov, chars); 411 redo: 412 addr = ops->map(pipe, buf, atomic); 413 error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic); 414 ops->unmap(pipe, buf, addr); 415 if (unlikely(error)) { 416 /* 417 * Just retry with the slow path if we failed. 418 */ 419 if (atomic) { 420 atomic = 0; 421 goto redo; 422 } 423 if (!ret) 424 ret = error; 425 break; 426 } 427 ret += chars; 428 buf->offset += chars; 429 buf->len -= chars; 430 431 /* Was it a packet buffer? Clean up and exit */ 432 if (buf->flags & PIPE_BUF_FLAG_PACKET) { 433 total_len = chars; 434 buf->len = 0; 435 } 436 437 if (!buf->len) { 438 buf->ops = NULL; 439 ops->release(pipe, buf); 440 curbuf = (curbuf + 1) & (pipe->buffers - 1); 441 pipe->curbuf = curbuf; 442 pipe->nrbufs = --bufs; 443 do_wakeup = 1; 444 } 445 total_len -= chars; 446 if (!total_len) 447 break; /* common path: read succeeded */ 448 } 449 if (bufs) /* More to do? */ 450 continue; 451 if (!pipe->writers) 452 break; 453 if (!pipe->waiting_writers) { 454 /* syscall merging: Usually we must not sleep 455 * if O_NONBLOCK is set, or if we got some data. 456 * But if a writer sleeps in kernel space, then 457 * we can wait for that data without violating POSIX. 458 */ 459 if (ret) 460 break; 461 if (filp->f_flags & O_NONBLOCK) { 462 ret = -EAGAIN; 463 break; 464 } 465 } 466 if (signal_pending(current)) { 467 if (!ret) 468 ret = -ERESTARTSYS; 469 break; 470 } 471 if (do_wakeup) { 472 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM); 473 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 474 } 475 pipe_wait(pipe); 476 } 477 __pipe_unlock(pipe); 478 479 /* Signal writers asynchronously that there is more room. */ 480 if (do_wakeup) { 481 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM); 482 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 483 } 484 if (ret > 0) 485 file_accessed(filp); 486 return ret; 487 } 488 489 static inline int is_packetized(struct file *file) 490 { 491 return (file->f_flags & O_DIRECT) != 0; 492 } 493 494 static ssize_t 495 pipe_write(struct kiocb *iocb, const struct iovec *_iov, 496 unsigned long nr_segs, loff_t ppos) 497 { 498 struct file *filp = iocb->ki_filp; 499 struct pipe_inode_info *pipe = filp->private_data; 500 ssize_t ret; 501 int do_wakeup; 502 struct iovec *iov = (struct iovec *)_iov; 503 size_t total_len; 504 ssize_t chars; 505 506 total_len = iov_length(iov, nr_segs); 507 /* Null write succeeds. */ 508 if (unlikely(total_len == 0)) 509 return 0; 510 511 do_wakeup = 0; 512 ret = 0; 513 __pipe_lock(pipe); 514 515 if (!pipe->readers) { 516 send_sig(SIGPIPE, current, 0); 517 ret = -EPIPE; 518 goto out; 519 } 520 521 /* We try to merge small writes */ 522 chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */ 523 if (pipe->nrbufs && chars != 0) { 524 int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & 525 (pipe->buffers - 1); 526 struct pipe_buffer *buf = pipe->bufs + lastbuf; 527 const struct pipe_buf_operations *ops = buf->ops; 528 int offset = buf->offset + buf->len; 529 530 if (ops->can_merge && offset + chars <= PAGE_SIZE) { 531 int error, atomic = 1; 532 void *addr; 533 534 error = ops->confirm(pipe, buf); 535 if (error) 536 goto out; 537 538 iov_fault_in_pages_read(iov, chars); 539 redo1: 540 addr = ops->map(pipe, buf, atomic); 541 error = pipe_iov_copy_from_user(offset + addr, iov, 542 chars, atomic); 543 ops->unmap(pipe, buf, addr); 544 ret = error; 545 do_wakeup = 1; 546 if (error) { 547 if (atomic) { 548 atomic = 0; 549 goto redo1; 550 } 551 goto out; 552 } 553 buf->len += chars; 554 total_len -= chars; 555 ret = chars; 556 if (!total_len) 557 goto out; 558 } 559 } 560 561 for (;;) { 562 int bufs; 563 564 if (!pipe->readers) { 565 send_sig(SIGPIPE, current, 0); 566 if (!ret) 567 ret = -EPIPE; 568 break; 569 } 570 bufs = pipe->nrbufs; 571 if (bufs < pipe->buffers) { 572 int newbuf = (pipe->curbuf + bufs) & (pipe->buffers-1); 573 struct pipe_buffer *buf = pipe->bufs + newbuf; 574 struct page *page = pipe->tmp_page; 575 char *src; 576 int error, atomic = 1; 577 578 if (!page) { 579 page = alloc_page(GFP_HIGHUSER); 580 if (unlikely(!page)) { 581 ret = ret ? : -ENOMEM; 582 break; 583 } 584 pipe->tmp_page = page; 585 } 586 /* Always wake up, even if the copy fails. Otherwise 587 * we lock up (O_NONBLOCK-)readers that sleep due to 588 * syscall merging. 589 * FIXME! Is this really true? 590 */ 591 do_wakeup = 1; 592 chars = PAGE_SIZE; 593 if (chars > total_len) 594 chars = total_len; 595 596 iov_fault_in_pages_read(iov, chars); 597 redo2: 598 if (atomic) 599 src = kmap_atomic(page); 600 else 601 src = kmap(page); 602 603 error = pipe_iov_copy_from_user(src, iov, chars, 604 atomic); 605 if (atomic) 606 kunmap_atomic(src); 607 else 608 kunmap(page); 609 610 if (unlikely(error)) { 611 if (atomic) { 612 atomic = 0; 613 goto redo2; 614 } 615 if (!ret) 616 ret = error; 617 break; 618 } 619 ret += chars; 620 621 /* Insert it into the buffer array */ 622 buf->page = page; 623 buf->ops = &anon_pipe_buf_ops; 624 buf->offset = 0; 625 buf->len = chars; 626 buf->flags = 0; 627 if (is_packetized(filp)) { 628 buf->ops = &packet_pipe_buf_ops; 629 buf->flags = PIPE_BUF_FLAG_PACKET; 630 } 631 pipe->nrbufs = ++bufs; 632 pipe->tmp_page = NULL; 633 634 total_len -= chars; 635 if (!total_len) 636 break; 637 } 638 if (bufs < pipe->buffers) 639 continue; 640 if (filp->f_flags & O_NONBLOCK) { 641 if (!ret) 642 ret = -EAGAIN; 643 break; 644 } 645 if (signal_pending(current)) { 646 if (!ret) 647 ret = -ERESTARTSYS; 648 break; 649 } 650 if (do_wakeup) { 651 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM); 652 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 653 do_wakeup = 0; 654 } 655 pipe->waiting_writers++; 656 pipe_wait(pipe); 657 pipe->waiting_writers--; 658 } 659 out: 660 __pipe_unlock(pipe); 661 if (do_wakeup) { 662 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM); 663 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 664 } 665 if (ret > 0) { 666 int err = file_update_time(filp); 667 if (err) 668 ret = err; 669 } 670 return ret; 671 } 672 673 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 674 { 675 struct pipe_inode_info *pipe = filp->private_data; 676 int count, buf, nrbufs; 677 678 switch (cmd) { 679 case FIONREAD: 680 __pipe_lock(pipe); 681 count = 0; 682 buf = pipe->curbuf; 683 nrbufs = pipe->nrbufs; 684 while (--nrbufs >= 0) { 685 count += pipe->bufs[buf].len; 686 buf = (buf+1) & (pipe->buffers - 1); 687 } 688 __pipe_unlock(pipe); 689 690 return put_user(count, (int __user *)arg); 691 default: 692 return -ENOIOCTLCMD; 693 } 694 } 695 696 /* No kernel lock held - fine */ 697 static unsigned int 698 pipe_poll(struct file *filp, poll_table *wait) 699 { 700 unsigned int mask; 701 struct pipe_inode_info *pipe = filp->private_data; 702 int nrbufs; 703 704 poll_wait(filp, &pipe->wait, wait); 705 706 /* Reading only -- no need for acquiring the semaphore. */ 707 nrbufs = pipe->nrbufs; 708 mask = 0; 709 if (filp->f_mode & FMODE_READ) { 710 mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0; 711 if (!pipe->writers && filp->f_version != pipe->w_counter) 712 mask |= POLLHUP; 713 } 714 715 if (filp->f_mode & FMODE_WRITE) { 716 mask |= (nrbufs < pipe->buffers) ? POLLOUT | POLLWRNORM : 0; 717 /* 718 * Most Unices do not set POLLERR for FIFOs but on Linux they 719 * behave exactly like pipes for poll(). 720 */ 721 if (!pipe->readers) 722 mask |= POLLERR; 723 } 724 725 return mask; 726 } 727 728 static int 729 pipe_release(struct inode *inode, struct file *file) 730 { 731 struct pipe_inode_info *pipe = inode->i_pipe; 732 int kill = 0; 733 734 __pipe_lock(pipe); 735 if (file->f_mode & FMODE_READ) 736 pipe->readers--; 737 if (file->f_mode & FMODE_WRITE) 738 pipe->writers--; 739 740 if (pipe->readers || pipe->writers) { 741 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM | POLLERR | POLLHUP); 742 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 743 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 744 } 745 spin_lock(&inode->i_lock); 746 if (!--pipe->files) { 747 inode->i_pipe = NULL; 748 kill = 1; 749 } 750 spin_unlock(&inode->i_lock); 751 __pipe_unlock(pipe); 752 753 if (kill) 754 free_pipe_info(pipe); 755 756 return 0; 757 } 758 759 static int 760 pipe_fasync(int fd, struct file *filp, int on) 761 { 762 struct pipe_inode_info *pipe = filp->private_data; 763 int retval = 0; 764 765 __pipe_lock(pipe); 766 if (filp->f_mode & FMODE_READ) 767 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); 768 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) { 769 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); 770 if (retval < 0 && (filp->f_mode & FMODE_READ)) 771 /* this can happen only if on == T */ 772 fasync_helper(-1, filp, 0, &pipe->fasync_readers); 773 } 774 __pipe_unlock(pipe); 775 return retval; 776 } 777 778 struct pipe_inode_info *alloc_pipe_info(void) 779 { 780 struct pipe_inode_info *pipe; 781 782 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); 783 if (pipe) { 784 pipe->bufs = kzalloc(sizeof(struct pipe_buffer) * PIPE_DEF_BUFFERS, GFP_KERNEL); 785 if (pipe->bufs) { 786 init_waitqueue_head(&pipe->wait); 787 pipe->r_counter = pipe->w_counter = 1; 788 pipe->buffers = PIPE_DEF_BUFFERS; 789 mutex_init(&pipe->mutex); 790 return pipe; 791 } 792 kfree(pipe); 793 } 794 795 return NULL; 796 } 797 798 void free_pipe_info(struct pipe_inode_info *pipe) 799 { 800 int i; 801 802 for (i = 0; i < pipe->buffers; i++) { 803 struct pipe_buffer *buf = pipe->bufs + i; 804 if (buf->ops) 805 buf->ops->release(pipe, buf); 806 } 807 if (pipe->tmp_page) 808 __free_page(pipe->tmp_page); 809 kfree(pipe->bufs); 810 kfree(pipe); 811 } 812 813 static struct vfsmount *pipe_mnt __read_mostly; 814 815 /* 816 * pipefs_dname() is called from d_path(). 817 */ 818 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen) 819 { 820 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]", 821 dentry->d_inode->i_ino); 822 } 823 824 static const struct dentry_operations pipefs_dentry_operations = { 825 .d_dname = pipefs_dname, 826 }; 827 828 static struct inode * get_pipe_inode(void) 829 { 830 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb); 831 struct pipe_inode_info *pipe; 832 833 if (!inode) 834 goto fail_inode; 835 836 inode->i_ino = get_next_ino(); 837 838 pipe = alloc_pipe_info(); 839 if (!pipe) 840 goto fail_iput; 841 842 inode->i_pipe = pipe; 843 pipe->files = 2; 844 pipe->readers = pipe->writers = 1; 845 inode->i_fop = &pipefifo_fops; 846 847 /* 848 * Mark the inode dirty from the very beginning, 849 * that way it will never be moved to the dirty 850 * list because "mark_inode_dirty()" will think 851 * that it already _is_ on the dirty list. 852 */ 853 inode->i_state = I_DIRTY; 854 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR; 855 inode->i_uid = current_fsuid(); 856 inode->i_gid = current_fsgid(); 857 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 858 859 return inode; 860 861 fail_iput: 862 iput(inode); 863 864 fail_inode: 865 return NULL; 866 } 867 868 int create_pipe_files(struct file **res, int flags) 869 { 870 int err; 871 struct inode *inode = get_pipe_inode(); 872 struct file *f; 873 struct path path; 874 static struct qstr name = { .name = "" }; 875 876 if (!inode) 877 return -ENFILE; 878 879 err = -ENOMEM; 880 path.dentry = d_alloc_pseudo(pipe_mnt->mnt_sb, &name); 881 if (!path.dentry) 882 goto err_inode; 883 path.mnt = mntget(pipe_mnt); 884 885 d_instantiate(path.dentry, inode); 886 887 err = -ENFILE; 888 f = alloc_file(&path, FMODE_WRITE, &pipefifo_fops); 889 if (IS_ERR(f)) 890 goto err_dentry; 891 892 f->f_flags = O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)); 893 f->private_data = inode->i_pipe; 894 895 res[0] = alloc_file(&path, FMODE_READ, &pipefifo_fops); 896 if (IS_ERR(res[0])) 897 goto err_file; 898 899 path_get(&path); 900 res[0]->private_data = inode->i_pipe; 901 res[0]->f_flags = O_RDONLY | (flags & O_NONBLOCK); 902 res[1] = f; 903 return 0; 904 905 err_file: 906 put_filp(f); 907 err_dentry: 908 free_pipe_info(inode->i_pipe); 909 path_put(&path); 910 return err; 911 912 err_inode: 913 free_pipe_info(inode->i_pipe); 914 iput(inode); 915 return err; 916 } 917 918 static int __do_pipe_flags(int *fd, struct file **files, int flags) 919 { 920 int error; 921 int fdw, fdr; 922 923 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT)) 924 return -EINVAL; 925 926 error = create_pipe_files(files, flags); 927 if (error) 928 return error; 929 930 error = get_unused_fd_flags(flags); 931 if (error < 0) 932 goto err_read_pipe; 933 fdr = error; 934 935 error = get_unused_fd_flags(flags); 936 if (error < 0) 937 goto err_fdr; 938 fdw = error; 939 940 audit_fd_pair(fdr, fdw); 941 fd[0] = fdr; 942 fd[1] = fdw; 943 return 0; 944 945 err_fdr: 946 put_unused_fd(fdr); 947 err_read_pipe: 948 fput(files[0]); 949 fput(files[1]); 950 return error; 951 } 952 953 int do_pipe_flags(int *fd, int flags) 954 { 955 struct file *files[2]; 956 int error = __do_pipe_flags(fd, files, flags); 957 if (!error) { 958 fd_install(fd[0], files[0]); 959 fd_install(fd[1], files[1]); 960 } 961 return error; 962 } 963 964 /* 965 * sys_pipe() is the normal C calling standard for creating 966 * a pipe. It's not the way Unix traditionally does this, though. 967 */ 968 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags) 969 { 970 struct file *files[2]; 971 int fd[2]; 972 int error; 973 974 error = __do_pipe_flags(fd, files, flags); 975 if (!error) { 976 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) { 977 fput(files[0]); 978 fput(files[1]); 979 put_unused_fd(fd[0]); 980 put_unused_fd(fd[1]); 981 error = -EFAULT; 982 } else { 983 fd_install(fd[0], files[0]); 984 fd_install(fd[1], files[1]); 985 } 986 } 987 return error; 988 } 989 990 SYSCALL_DEFINE1(pipe, int __user *, fildes) 991 { 992 return sys_pipe2(fildes, 0); 993 } 994 995 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) 996 { 997 int cur = *cnt; 998 999 while (cur == *cnt) { 1000 pipe_wait(pipe); 1001 if (signal_pending(current)) 1002 break; 1003 } 1004 return cur == *cnt ? -ERESTARTSYS : 0; 1005 } 1006 1007 static void wake_up_partner(struct pipe_inode_info *pipe) 1008 { 1009 wake_up_interruptible(&pipe->wait); 1010 } 1011 1012 static int fifo_open(struct inode *inode, struct file *filp) 1013 { 1014 struct pipe_inode_info *pipe; 1015 bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC; 1016 int kill = 0; 1017 int ret; 1018 1019 filp->f_version = 0; 1020 1021 spin_lock(&inode->i_lock); 1022 if (inode->i_pipe) { 1023 pipe = inode->i_pipe; 1024 pipe->files++; 1025 spin_unlock(&inode->i_lock); 1026 } else { 1027 spin_unlock(&inode->i_lock); 1028 pipe = alloc_pipe_info(); 1029 if (!pipe) 1030 return -ENOMEM; 1031 pipe->files = 1; 1032 spin_lock(&inode->i_lock); 1033 if (unlikely(inode->i_pipe)) { 1034 inode->i_pipe->files++; 1035 spin_unlock(&inode->i_lock); 1036 free_pipe_info(pipe); 1037 pipe = inode->i_pipe; 1038 } else { 1039 inode->i_pipe = pipe; 1040 spin_unlock(&inode->i_lock); 1041 } 1042 } 1043 filp->private_data = pipe; 1044 /* OK, we have a pipe and it's pinned down */ 1045 1046 __pipe_lock(pipe); 1047 1048 /* We can only do regular read/write on fifos */ 1049 filp->f_mode &= (FMODE_READ | FMODE_WRITE); 1050 1051 switch (filp->f_mode) { 1052 case FMODE_READ: 1053 /* 1054 * O_RDONLY 1055 * POSIX.1 says that O_NONBLOCK means return with the FIFO 1056 * opened, even when there is no process writing the FIFO. 1057 */ 1058 pipe->r_counter++; 1059 if (pipe->readers++ == 0) 1060 wake_up_partner(pipe); 1061 1062 if (!is_pipe && !pipe->writers) { 1063 if ((filp->f_flags & O_NONBLOCK)) { 1064 /* suppress POLLHUP until we have 1065 * seen a writer */ 1066 filp->f_version = pipe->w_counter; 1067 } else { 1068 if (wait_for_partner(pipe, &pipe->w_counter)) 1069 goto err_rd; 1070 } 1071 } 1072 break; 1073 1074 case FMODE_WRITE: 1075 /* 1076 * O_WRONLY 1077 * POSIX.1 says that O_NONBLOCK means return -1 with 1078 * errno=ENXIO when there is no process reading the FIFO. 1079 */ 1080 ret = -ENXIO; 1081 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) 1082 goto err; 1083 1084 pipe->w_counter++; 1085 if (!pipe->writers++) 1086 wake_up_partner(pipe); 1087 1088 if (!is_pipe && !pipe->readers) { 1089 if (wait_for_partner(pipe, &pipe->r_counter)) 1090 goto err_wr; 1091 } 1092 break; 1093 1094 case FMODE_READ | FMODE_WRITE: 1095 /* 1096 * O_RDWR 1097 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set. 1098 * This implementation will NEVER block on a O_RDWR open, since 1099 * the process can at least talk to itself. 1100 */ 1101 1102 pipe->readers++; 1103 pipe->writers++; 1104 pipe->r_counter++; 1105 pipe->w_counter++; 1106 if (pipe->readers == 1 || pipe->writers == 1) 1107 wake_up_partner(pipe); 1108 break; 1109 1110 default: 1111 ret = -EINVAL; 1112 goto err; 1113 } 1114 1115 /* Ok! */ 1116 __pipe_unlock(pipe); 1117 return 0; 1118 1119 err_rd: 1120 if (!--pipe->readers) 1121 wake_up_interruptible(&pipe->wait); 1122 ret = -ERESTARTSYS; 1123 goto err; 1124 1125 err_wr: 1126 if (!--pipe->writers) 1127 wake_up_interruptible(&pipe->wait); 1128 ret = -ERESTARTSYS; 1129 goto err; 1130 1131 err: 1132 spin_lock(&inode->i_lock); 1133 if (!--pipe->files) { 1134 inode->i_pipe = NULL; 1135 kill = 1; 1136 } 1137 spin_unlock(&inode->i_lock); 1138 __pipe_unlock(pipe); 1139 if (kill) 1140 free_pipe_info(pipe); 1141 return ret; 1142 } 1143 1144 const struct file_operations pipefifo_fops = { 1145 .open = fifo_open, 1146 .llseek = no_llseek, 1147 .read = do_sync_read, 1148 .aio_read = pipe_read, 1149 .write = do_sync_write, 1150 .aio_write = pipe_write, 1151 .poll = pipe_poll, 1152 .unlocked_ioctl = pipe_ioctl, 1153 .release = pipe_release, 1154 .fasync = pipe_fasync, 1155 }; 1156 1157 /* 1158 * Allocate a new array of pipe buffers and copy the info over. Returns the 1159 * pipe size if successful, or return -ERROR on error. 1160 */ 1161 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long nr_pages) 1162 { 1163 struct pipe_buffer *bufs; 1164 1165 /* 1166 * We can shrink the pipe, if arg >= pipe->nrbufs. Since we don't 1167 * expect a lot of shrink+grow operations, just free and allocate 1168 * again like we would do for growing. If the pipe currently 1169 * contains more buffers than arg, then return busy. 1170 */ 1171 if (nr_pages < pipe->nrbufs) 1172 return -EBUSY; 1173 1174 bufs = kcalloc(nr_pages, sizeof(*bufs), GFP_KERNEL | __GFP_NOWARN); 1175 if (unlikely(!bufs)) 1176 return -ENOMEM; 1177 1178 /* 1179 * The pipe array wraps around, so just start the new one at zero 1180 * and adjust the indexes. 1181 */ 1182 if (pipe->nrbufs) { 1183 unsigned int tail; 1184 unsigned int head; 1185 1186 tail = pipe->curbuf + pipe->nrbufs; 1187 if (tail < pipe->buffers) 1188 tail = 0; 1189 else 1190 tail &= (pipe->buffers - 1); 1191 1192 head = pipe->nrbufs - tail; 1193 if (head) 1194 memcpy(bufs, pipe->bufs + pipe->curbuf, head * sizeof(struct pipe_buffer)); 1195 if (tail) 1196 memcpy(bufs + head, pipe->bufs, tail * sizeof(struct pipe_buffer)); 1197 } 1198 1199 pipe->curbuf = 0; 1200 kfree(pipe->bufs); 1201 pipe->bufs = bufs; 1202 pipe->buffers = nr_pages; 1203 return nr_pages * PAGE_SIZE; 1204 } 1205 1206 /* 1207 * Currently we rely on the pipe array holding a power-of-2 number 1208 * of pages. 1209 */ 1210 static inline unsigned int round_pipe_size(unsigned int size) 1211 { 1212 unsigned long nr_pages; 1213 1214 nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1215 return roundup_pow_of_two(nr_pages) << PAGE_SHIFT; 1216 } 1217 1218 /* 1219 * This should work even if CONFIG_PROC_FS isn't set, as proc_dointvec_minmax 1220 * will return an error. 1221 */ 1222 int pipe_proc_fn(struct ctl_table *table, int write, void __user *buf, 1223 size_t *lenp, loff_t *ppos) 1224 { 1225 int ret; 1226 1227 ret = proc_dointvec_minmax(table, write, buf, lenp, ppos); 1228 if (ret < 0 || !write) 1229 return ret; 1230 1231 pipe_max_size = round_pipe_size(pipe_max_size); 1232 return ret; 1233 } 1234 1235 /* 1236 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same 1237 * location, so checking ->i_pipe is not enough to verify that this is a 1238 * pipe. 1239 */ 1240 struct pipe_inode_info *get_pipe_info(struct file *file) 1241 { 1242 return file->f_op == &pipefifo_fops ? file->private_data : NULL; 1243 } 1244 1245 long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg) 1246 { 1247 struct pipe_inode_info *pipe; 1248 long ret; 1249 1250 pipe = get_pipe_info(file); 1251 if (!pipe) 1252 return -EBADF; 1253 1254 __pipe_lock(pipe); 1255 1256 switch (cmd) { 1257 case F_SETPIPE_SZ: { 1258 unsigned int size, nr_pages; 1259 1260 size = round_pipe_size(arg); 1261 nr_pages = size >> PAGE_SHIFT; 1262 1263 ret = -EINVAL; 1264 if (!nr_pages) 1265 goto out; 1266 1267 if (!capable(CAP_SYS_RESOURCE) && size > pipe_max_size) { 1268 ret = -EPERM; 1269 goto out; 1270 } 1271 ret = pipe_set_size(pipe, nr_pages); 1272 break; 1273 } 1274 case F_GETPIPE_SZ: 1275 ret = pipe->buffers * PAGE_SIZE; 1276 break; 1277 default: 1278 ret = -EINVAL; 1279 break; 1280 } 1281 1282 out: 1283 __pipe_unlock(pipe); 1284 return ret; 1285 } 1286 1287 static const struct super_operations pipefs_ops = { 1288 .destroy_inode = free_inode_nonrcu, 1289 .statfs = simple_statfs, 1290 }; 1291 1292 /* 1293 * pipefs should _never_ be mounted by userland - too much of security hassle, 1294 * no real gain from having the whole whorehouse mounted. So we don't need 1295 * any operations on the root directory. However, we need a non-trivial 1296 * d_name - pipe: will go nicely and kill the special-casing in procfs. 1297 */ 1298 static struct dentry *pipefs_mount(struct file_system_type *fs_type, 1299 int flags, const char *dev_name, void *data) 1300 { 1301 return mount_pseudo(fs_type, "pipe:", &pipefs_ops, 1302 &pipefs_dentry_operations, PIPEFS_MAGIC); 1303 } 1304 1305 static struct file_system_type pipe_fs_type = { 1306 .name = "pipefs", 1307 .mount = pipefs_mount, 1308 .kill_sb = kill_anon_super, 1309 }; 1310 1311 static int __init init_pipe_fs(void) 1312 { 1313 int err = register_filesystem(&pipe_fs_type); 1314 1315 if (!err) { 1316 pipe_mnt = kern_mount(&pipe_fs_type); 1317 if (IS_ERR(pipe_mnt)) { 1318 err = PTR_ERR(pipe_mnt); 1319 unregister_filesystem(&pipe_fs_type); 1320 } 1321 } 1322 return err; 1323 } 1324 1325 fs_initcall(init_pipe_fs); 1326