1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * "splice": joining two ropes together by interweaving their strands. 4 * 5 * This is the "extended pipe" functionality, where a pipe is used as 6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 7 * buffer that you can use to transfer data from one end to the other. 8 * 9 * The traditional unix read/write is extended with a "splice()" operation 10 * that transfers data buffers to or from a pipe buffer. 11 * 12 * Named by Larry McVoy, original implementation from Linus, extended by 13 * Jens to support splicing to files, network, direct splicing, etc and 14 * fixing lots of bugs. 15 * 16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 18 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 19 * 20 */ 21 #include <linux/bvec.h> 22 #include <linux/fs.h> 23 #include <linux/file.h> 24 #include <linux/pagemap.h> 25 #include <linux/splice.h> 26 #include <linux/memcontrol.h> 27 #include <linux/mm_inline.h> 28 #include <linux/swap.h> 29 #include <linux/writeback.h> 30 #include <linux/export.h> 31 #include <linux/syscalls.h> 32 #include <linux/uio.h> 33 #include <linux/fsnotify.h> 34 #include <linux/security.h> 35 #include <linux/gfp.h> 36 #include <linux/net.h> 37 #include <linux/socket.h> 38 #include <linux/sched/signal.h> 39 40 #include "internal.h" 41 42 /* 43 * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to 44 * indicate they support non-blocking reads or writes, we must clear it 45 * here if set to avoid blocking other users of this pipe if splice is 46 * being done on it. 47 */ 48 static noinline void pipe_clear_nowait(struct file *file) 49 { 50 fmode_t fmode = READ_ONCE(file->f_mode); 51 52 do { 53 if (!(fmode & FMODE_NOWAIT)) 54 break; 55 } while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT)); 56 } 57 58 /* 59 * Check whether the contents of buf is OK to access. Since the content 60 * is a page cache page, IO may be in flight. 61 */ 62 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 63 struct pipe_buffer *buf) 64 { 65 struct folio *folio = page_folio(buf->page); 66 int err; 67 68 if (!folio_test_uptodate(folio)) { 69 folio_lock(folio); 70 71 /* 72 * Folio got truncated/unhashed. This will cause a 0-byte 73 * splice, if this is the first page. 74 */ 75 if (!folio->mapping) { 76 err = -ENODATA; 77 goto error; 78 } 79 80 /* 81 * Uh oh, read-error from disk. 82 */ 83 if (!folio_test_uptodate(folio)) { 84 err = -EIO; 85 goto error; 86 } 87 88 /* Folio is ok after all, we are done */ 89 folio_unlock(folio); 90 } 91 92 return 0; 93 error: 94 folio_unlock(folio); 95 return err; 96 } 97 98 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 99 .confirm = page_cache_pipe_buf_confirm, 100 .release = generic_pipe_buf_release, 101 .get = generic_pipe_buf_get, 102 }; 103 104 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 105 .release = generic_pipe_buf_release, 106 .get = generic_pipe_buf_get, 107 }; 108 109 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 110 { 111 if (wq_has_sleeper(&pipe->rd_wait)) 112 wake_up_interruptible_poll(&pipe->rd_wait, 113 EPOLLIN | EPOLLRDNORM); 114 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 115 } 116 117 /** 118 * splice_to_pipe - fill passed data into a pipe 119 * @pipe: pipe to fill 120 * @spd: data to fill 121 * 122 * Description: 123 * @spd contains a map of pages and len/offset tuples, along with 124 * the struct pipe_buf_operations associated with these pages. This 125 * function will link that data to the pipe. 126 * 127 */ 128 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 129 struct splice_pipe_desc *spd) 130 { 131 unsigned int spd_pages = spd->nr_pages; 132 unsigned int tail = pipe->tail; 133 unsigned int head = pipe->head; 134 ssize_t ret = 0; 135 int page_nr = 0; 136 137 if (!spd_pages) 138 return 0; 139 140 if (unlikely(!pipe->readers)) { 141 send_sig(SIGPIPE, current, 0); 142 ret = -EPIPE; 143 goto out; 144 } 145 146 while (!pipe_full(head, tail, pipe->max_usage)) { 147 struct pipe_buffer *buf = pipe_buf(pipe, head); 148 149 buf->page = spd->pages[page_nr]; 150 buf->offset = spd->partial[page_nr].offset; 151 buf->len = spd->partial[page_nr].len; 152 buf->private = spd->partial[page_nr].private; 153 buf->ops = spd->ops; 154 buf->flags = 0; 155 156 head++; 157 pipe->head = head; 158 page_nr++; 159 ret += buf->len; 160 161 if (!--spd->nr_pages) 162 break; 163 } 164 165 if (!ret) 166 ret = -EAGAIN; 167 168 out: 169 while (page_nr < spd_pages) 170 spd->spd_release(spd, page_nr++); 171 172 return ret; 173 } 174 EXPORT_SYMBOL_GPL(splice_to_pipe); 175 176 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 177 { 178 unsigned int head = pipe->head; 179 unsigned int tail = pipe->tail; 180 int ret; 181 182 if (unlikely(!pipe->readers)) { 183 send_sig(SIGPIPE, current, 0); 184 ret = -EPIPE; 185 } else if (pipe_full(head, tail, pipe->max_usage)) { 186 ret = -EAGAIN; 187 } else { 188 *pipe_buf(pipe, head) = *buf; 189 pipe->head = head + 1; 190 return buf->len; 191 } 192 pipe_buf_release(pipe, buf); 193 return ret; 194 } 195 EXPORT_SYMBOL(add_to_pipe); 196 197 /* 198 * Check if we need to grow the arrays holding pages and partial page 199 * descriptions. 200 */ 201 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 202 { 203 unsigned int max_usage = READ_ONCE(pipe->max_usage); 204 205 spd->nr_pages_max = max_usage; 206 if (max_usage <= PIPE_DEF_BUFFERS) 207 return 0; 208 209 spd->pages = kmalloc_objs(struct page *, max_usage); 210 spd->partial = kmalloc_objs(struct partial_page, max_usage); 211 212 if (spd->pages && spd->partial) 213 return 0; 214 215 kfree(spd->pages); 216 kfree(spd->partial); 217 return -ENOMEM; 218 } 219 220 void splice_shrink_spd(struct splice_pipe_desc *spd) 221 { 222 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 223 return; 224 225 kfree(spd->pages); 226 kfree(spd->partial); 227 } 228 229 /** 230 * copy_splice_read - Copy data from a file and splice the copy into a pipe 231 * @in: The file to read from 232 * @ppos: Pointer to the file position to read from 233 * @pipe: The pipe to splice into 234 * @len: The amount to splice 235 * @flags: The SPLICE_F_* flags 236 * 237 * This function allocates a bunch of pages sufficient to hold the requested 238 * amount of data (but limited by the remaining pipe capacity), passes it to 239 * the file's ->read_iter() to read into and then splices the used pages into 240 * the pipe. 241 * 242 * Return: On success, the number of bytes read will be returned and *@ppos 243 * will be updated if appropriate; 0 will be returned if there is no more data 244 * to be read; -EAGAIN will be returned if the pipe had no space, and some 245 * other negative error code will be returned on error. A short read may occur 246 * if the pipe has insufficient space, we reach the end of the data or we hit a 247 * hole. 248 */ 249 ssize_t copy_splice_read(struct file *in, loff_t *ppos, 250 struct pipe_inode_info *pipe, 251 size_t len, unsigned int flags) 252 { 253 struct iov_iter to; 254 struct bio_vec *bv; 255 struct kiocb kiocb; 256 struct page **pages; 257 ssize_t ret; 258 size_t used, npages, chunk, remain, keep = 0; 259 int i; 260 261 /* Work out how much data we can actually add into the pipe */ 262 used = pipe_buf_usage(pipe); 263 npages = max_t(ssize_t, pipe->max_usage - used, 0); 264 len = min_t(size_t, len, npages * PAGE_SIZE); 265 npages = DIV_ROUND_UP(len, PAGE_SIZE); 266 267 bv = kzalloc(array_size(npages, sizeof(bv[0])) + 268 array_size(npages, sizeof(struct page *)), GFP_KERNEL); 269 if (!bv) 270 return -ENOMEM; 271 272 pages = (struct page **)(bv + npages); 273 npages = alloc_pages_bulk(GFP_USER, npages, pages); 274 if (!npages) { 275 kfree(bv); 276 return -ENOMEM; 277 } 278 279 remain = len = min_t(size_t, len, npages * PAGE_SIZE); 280 281 for (i = 0; i < npages; i++) { 282 chunk = min_t(size_t, PAGE_SIZE, remain); 283 bv[i].bv_page = pages[i]; 284 bv[i].bv_offset = 0; 285 bv[i].bv_len = chunk; 286 remain -= chunk; 287 } 288 289 /* Do the I/O */ 290 iov_iter_bvec(&to, ITER_DEST, bv, npages, len); 291 init_sync_kiocb(&kiocb, in); 292 kiocb.ki_pos = *ppos; 293 ret = in->f_op->read_iter(&kiocb, &to); 294 295 if (ret > 0) { 296 keep = DIV_ROUND_UP(ret, PAGE_SIZE); 297 *ppos = kiocb.ki_pos; 298 } 299 300 /* 301 * Callers of ->splice_read() expect -EAGAIN on "can't put anything in 302 * there", rather than -EFAULT. 303 */ 304 if (ret == -EFAULT) 305 ret = -EAGAIN; 306 307 /* Free any pages that didn't get touched at all. */ 308 if (keep < npages) 309 release_pages(pages + keep, npages - keep); 310 311 /* Push the remaining pages into the pipe. */ 312 remain = ret; 313 for (i = 0; i < keep; i++) { 314 struct pipe_buffer *buf = pipe_head_buf(pipe); 315 316 chunk = min_t(size_t, remain, PAGE_SIZE); 317 *buf = (struct pipe_buffer) { 318 .ops = &default_pipe_buf_ops, 319 .page = bv[i].bv_page, 320 .offset = 0, 321 .len = chunk, 322 }; 323 pipe->head++; 324 remain -= chunk; 325 } 326 327 kfree(bv); 328 return ret; 329 } 330 EXPORT_SYMBOL(copy_splice_read); 331 332 const struct pipe_buf_operations default_pipe_buf_ops = { 333 .release = generic_pipe_buf_release, 334 .get = generic_pipe_buf_get, 335 }; 336 EXPORT_SYMBOL(default_pipe_buf_ops); 337 338 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 339 { 340 if (wq_has_sleeper(&pipe->wr_wait)) 341 wake_up_interruptible_poll(&pipe->wr_wait, 342 EPOLLOUT | EPOLLWRNORM); 343 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 344 } 345 346 /** 347 * splice_from_pipe_feed - feed available data from a pipe to a file 348 * @pipe: pipe to splice from 349 * @sd: information to @actor 350 * @actor: handler that splices the data 351 * 352 * Description: 353 * This function loops over the pipe and calls @actor to do the 354 * actual moving of a single struct pipe_buffer to the desired 355 * destination. It returns when there's no more buffers left in 356 * the pipe or if the requested number of bytes (@sd->total_len) 357 * have been copied. It returns a positive number (one) if the 358 * pipe needs to be filled with more data, zero if the required 359 * number of bytes have been copied and -errno on error. 360 * 361 * This, together with splice_from_pipe_{begin,end,next}, may be 362 * used to implement the functionality of __splice_from_pipe() when 363 * locking is required around copying the pipe buffers to the 364 * destination. 365 */ 366 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 367 splice_actor *actor) 368 { 369 unsigned int head = pipe->head; 370 unsigned int tail = pipe->tail; 371 int ret; 372 373 while (!pipe_empty(head, tail)) { 374 struct pipe_buffer *buf = pipe_buf(pipe, tail); 375 376 sd->len = buf->len; 377 if (sd->len > sd->total_len) 378 sd->len = sd->total_len; 379 380 ret = pipe_buf_confirm(pipe, buf); 381 if (unlikely(ret)) { 382 if (ret == -ENODATA) 383 ret = 0; 384 return ret; 385 } 386 387 ret = actor(pipe, buf, sd); 388 if (ret <= 0) 389 return ret; 390 391 buf->offset += ret; 392 buf->len -= ret; 393 394 sd->num_spliced += ret; 395 sd->len -= ret; 396 sd->pos += ret; 397 sd->total_len -= ret; 398 399 if (!buf->len) { 400 pipe_buf_release(pipe, buf); 401 tail++; 402 pipe->tail = tail; 403 if (pipe->files) 404 sd->need_wakeup = true; 405 } 406 407 if (!sd->total_len) 408 return 0; 409 } 410 411 return 1; 412 } 413 414 /* We know we have a pipe buffer, but maybe it's empty? */ 415 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe) 416 { 417 unsigned int tail = pipe->tail; 418 struct pipe_buffer *buf = pipe_buf(pipe, tail); 419 420 if (unlikely(!buf->len)) { 421 pipe_buf_release(pipe, buf); 422 pipe->tail = tail+1; 423 return true; 424 } 425 426 return false; 427 } 428 429 /** 430 * splice_from_pipe_next - wait for some data to splice from 431 * @pipe: pipe to splice from 432 * @sd: information about the splice operation 433 * 434 * Description: 435 * This function will wait for some data and return a positive 436 * value (one) if pipe buffers are available. It will return zero 437 * or -errno if no more data needs to be spliced. 438 */ 439 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 440 { 441 /* 442 * Check for signal early to make process killable when there are 443 * always buffers available 444 */ 445 if (signal_pending(current)) 446 return -ERESTARTSYS; 447 448 repeat: 449 while (pipe_is_empty(pipe)) { 450 if (!pipe->writers) 451 return 0; 452 453 if (sd->num_spliced) 454 return 0; 455 456 if (sd->flags & SPLICE_F_NONBLOCK) 457 return -EAGAIN; 458 459 if (signal_pending(current)) 460 return -ERESTARTSYS; 461 462 if (sd->need_wakeup) { 463 wakeup_pipe_writers(pipe); 464 sd->need_wakeup = false; 465 } 466 467 pipe_wait_readable(pipe); 468 } 469 470 if (eat_empty_buffer(pipe)) 471 goto repeat; 472 473 return 1; 474 } 475 476 /** 477 * splice_from_pipe_begin - start splicing from pipe 478 * @sd: information about the splice operation 479 * 480 * Description: 481 * This function should be called before a loop containing 482 * splice_from_pipe_next() and splice_from_pipe_feed() to 483 * initialize the necessary fields of @sd. 484 */ 485 static void splice_from_pipe_begin(struct splice_desc *sd) 486 { 487 sd->num_spliced = 0; 488 sd->need_wakeup = false; 489 } 490 491 /** 492 * splice_from_pipe_end - finish splicing from pipe 493 * @pipe: pipe to splice from 494 * @sd: information about the splice operation 495 * 496 * Description: 497 * This function will wake up pipe writers if necessary. It should 498 * be called after a loop containing splice_from_pipe_next() and 499 * splice_from_pipe_feed(). 500 */ 501 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 502 { 503 if (sd->need_wakeup) 504 wakeup_pipe_writers(pipe); 505 } 506 507 /** 508 * __splice_from_pipe - splice data from a pipe to given actor 509 * @pipe: pipe to splice from 510 * @sd: information to @actor 511 * @actor: handler that splices the data 512 * 513 * Description: 514 * This function does little more than loop over the pipe and call 515 * @actor to do the actual moving of a single struct pipe_buffer to 516 * the desired destination. See pipe_to_file, pipe_to_sendmsg, or 517 * pipe_to_user. 518 * 519 */ 520 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 521 splice_actor *actor) 522 { 523 int ret; 524 525 splice_from_pipe_begin(sd); 526 do { 527 cond_resched(); 528 ret = splice_from_pipe_next(pipe, sd); 529 if (ret > 0) 530 ret = splice_from_pipe_feed(pipe, sd, actor); 531 } while (ret > 0); 532 splice_from_pipe_end(pipe, sd); 533 534 return sd->num_spliced ? sd->num_spliced : ret; 535 } 536 EXPORT_SYMBOL(__splice_from_pipe); 537 538 /** 539 * splice_from_pipe - splice data from a pipe to a file 540 * @pipe: pipe to splice from 541 * @out: file to splice to 542 * @ppos: position in @out 543 * @len: how many bytes to splice 544 * @flags: splice modifier flags 545 * @actor: handler that splices the data 546 * 547 * Description: 548 * See __splice_from_pipe. This function locks the pipe inode, 549 * otherwise it's identical to __splice_from_pipe(). 550 * 551 */ 552 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 553 loff_t *ppos, size_t len, unsigned int flags, 554 splice_actor *actor) 555 { 556 ssize_t ret; 557 struct splice_desc sd = { 558 .total_len = len, 559 .flags = flags, 560 .pos = *ppos, 561 .u.file = out, 562 }; 563 564 pipe_lock(pipe); 565 ret = __splice_from_pipe(pipe, &sd, actor); 566 pipe_unlock(pipe); 567 568 return ret; 569 } 570 571 /** 572 * iter_file_splice_write - splice data from a pipe to a file 573 * @pipe: pipe info 574 * @out: file to write to 575 * @ppos: position in @out 576 * @len: number of bytes to splice 577 * @flags: splice modifier flags 578 * 579 * Description: 580 * Will either move or copy pages (determined by @flags options) from 581 * the given pipe inode to the given file. 582 * This one is ->write_iter-based. 583 * 584 */ 585 ssize_t 586 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 587 loff_t *ppos, size_t len, unsigned int flags) 588 { 589 struct splice_desc sd = { 590 .total_len = len, 591 .flags = flags, 592 .pos = *ppos, 593 .u.file = out, 594 }; 595 int nbufs = pipe->max_usage; 596 struct bio_vec *array; 597 ssize_t ret; 598 599 if (!out->f_op->write_iter) 600 return -EINVAL; 601 602 array = kzalloc_objs(struct bio_vec, nbufs); 603 if (unlikely(!array)) 604 return -ENOMEM; 605 606 pipe_lock(pipe); 607 608 splice_from_pipe_begin(&sd); 609 while (sd.total_len) { 610 struct kiocb kiocb; 611 struct iov_iter from; 612 unsigned int head, tail; 613 size_t left; 614 int n; 615 616 ret = splice_from_pipe_next(pipe, &sd); 617 if (ret <= 0) 618 break; 619 620 if (unlikely(nbufs < pipe->max_usage)) { 621 kfree(array); 622 nbufs = pipe->max_usage; 623 array = kzalloc_objs(struct bio_vec, nbufs); 624 if (!array) { 625 ret = -ENOMEM; 626 break; 627 } 628 } 629 630 head = pipe->head; 631 tail = pipe->tail; 632 633 /* build the vector */ 634 left = sd.total_len; 635 for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) { 636 struct pipe_buffer *buf = pipe_buf(pipe, tail); 637 size_t this_len = buf->len; 638 639 /* zero-length bvecs are not supported, skip them */ 640 if (!this_len) 641 continue; 642 this_len = min(this_len, left); 643 644 ret = pipe_buf_confirm(pipe, buf); 645 if (unlikely(ret)) { 646 if (ret == -ENODATA) 647 ret = 0; 648 goto done; 649 } 650 651 bvec_set_page(&array[n], buf->page, this_len, 652 buf->offset); 653 left -= this_len; 654 n++; 655 } 656 657 iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left); 658 init_sync_kiocb(&kiocb, out); 659 kiocb.ki_pos = sd.pos; 660 ret = out->f_op->write_iter(&kiocb, &from); 661 sd.pos = kiocb.ki_pos; 662 if (ret <= 0) 663 break; 664 if (unlikely(ret > sd.total_len - left)) { 665 WARN_ONCE(1, 666 "Splice Exceeded! ret=%zd tot=%zu left=%zu\n", 667 ret, sd.total_len, left); 668 ret = sd.total_len - left; 669 } 670 671 sd.num_spliced += ret; 672 sd.total_len -= ret; 673 *ppos = sd.pos; 674 675 /* dismiss the fully eaten buffers, adjust the partial one */ 676 tail = pipe->tail; 677 while (ret) { 678 struct pipe_buffer *buf = pipe_buf(pipe, tail); 679 if (ret >= buf->len) { 680 ret -= buf->len; 681 buf->len = 0; 682 pipe_buf_release(pipe, buf); 683 tail++; 684 pipe->tail = tail; 685 if (pipe->files) 686 sd.need_wakeup = true; 687 } else { 688 buf->offset += ret; 689 buf->len -= ret; 690 ret = 0; 691 } 692 } 693 } 694 done: 695 kfree(array); 696 splice_from_pipe_end(pipe, &sd); 697 698 pipe_unlock(pipe); 699 700 if (sd.num_spliced) 701 ret = sd.num_spliced; 702 703 return ret; 704 } 705 706 EXPORT_SYMBOL(iter_file_splice_write); 707 708 #ifdef CONFIG_NET 709 /** 710 * splice_to_socket - splice data from a pipe to a socket 711 * @pipe: pipe to splice from 712 * @out: socket to write to 713 * @ppos: position in @out 714 * @len: number of bytes to splice 715 * @flags: splice modifier flags 716 * 717 * Description: 718 * Will send @len bytes from the pipe to a network socket. No data copying 719 * is involved. 720 * 721 */ 722 ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, 723 loff_t *ppos, size_t len, unsigned int flags) 724 { 725 struct socket *sock = sock_from_file(out); 726 struct bio_vec bvec[16]; 727 struct msghdr msg = {}; 728 ssize_t ret = 0; 729 size_t spliced = 0; 730 bool need_wakeup = false; 731 732 pipe_lock(pipe); 733 734 while (len > 0) { 735 unsigned int head, tail, bc = 0; 736 size_t remain = len; 737 738 /* 739 * Check for signal early to make process killable when there 740 * are always buffers available 741 */ 742 ret = -ERESTARTSYS; 743 if (signal_pending(current)) 744 break; 745 746 while (pipe_is_empty(pipe)) { 747 ret = 0; 748 if (!pipe->writers) 749 goto out; 750 751 if (spliced) 752 goto out; 753 754 ret = -EAGAIN; 755 if (flags & SPLICE_F_NONBLOCK) 756 goto out; 757 758 ret = -ERESTARTSYS; 759 if (signal_pending(current)) 760 goto out; 761 762 if (need_wakeup) { 763 wakeup_pipe_writers(pipe); 764 need_wakeup = false; 765 } 766 767 pipe_wait_readable(pipe); 768 } 769 770 head = pipe->head; 771 tail = pipe->tail; 772 773 while (!pipe_empty(head, tail)) { 774 struct pipe_buffer *buf = pipe_buf(pipe, tail); 775 size_t seg; 776 777 if (!buf->len) { 778 tail++; 779 continue; 780 } 781 782 seg = min_t(size_t, remain, buf->len); 783 784 ret = pipe_buf_confirm(pipe, buf); 785 if (unlikely(ret)) { 786 if (ret == -ENODATA) 787 ret = 0; 788 break; 789 } 790 791 bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset); 792 remain -= seg; 793 tail++; 794 if (remain == 0 || bc >= ARRAY_SIZE(bvec)) 795 break; 796 } 797 798 if (!bc) 799 break; 800 801 msg.msg_flags = MSG_SPLICE_PAGES; 802 if (flags & SPLICE_F_MORE) 803 msg.msg_flags |= MSG_MORE; 804 if (remain && pipe_occupancy(pipe->head, tail) > 0) 805 msg.msg_flags |= MSG_MORE; 806 if (out->f_flags & O_NONBLOCK) 807 msg.msg_flags |= MSG_DONTWAIT; 808 809 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bc, 810 len - remain); 811 ret = sock_sendmsg(sock, &msg); 812 if (ret <= 0) 813 break; 814 815 spliced += ret; 816 len -= ret; 817 tail = pipe->tail; 818 while (ret > 0) { 819 struct pipe_buffer *buf = pipe_buf(pipe, tail); 820 size_t seg = min_t(size_t, ret, buf->len); 821 822 buf->offset += seg; 823 buf->len -= seg; 824 ret -= seg; 825 826 if (!buf->len) { 827 pipe_buf_release(pipe, buf); 828 tail++; 829 } 830 } 831 832 if (tail != pipe->tail) { 833 pipe->tail = tail; 834 if (pipe->files) 835 need_wakeup = true; 836 } 837 } 838 839 out: 840 pipe_unlock(pipe); 841 if (need_wakeup) 842 wakeup_pipe_writers(pipe); 843 return spliced ?: ret; 844 } 845 #endif 846 847 static int warn_unsupported(struct file *file, const char *op) 848 { 849 pr_debug_ratelimited( 850 "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n", 851 op, file, current->pid, current->comm); 852 return -EINVAL; 853 } 854 855 /* 856 * Attempt to initiate a splice from pipe to file. 857 */ 858 static ssize_t do_splice_from(struct pipe_inode_info *pipe, struct file *out, 859 loff_t *ppos, size_t len, unsigned int flags) 860 { 861 if (unlikely(!out->f_op->splice_write)) 862 return warn_unsupported(out, "write"); 863 return out->f_op->splice_write(pipe, out, ppos, len, flags); 864 } 865 866 /* 867 * Indicate to the caller that there was a premature EOF when reading from the 868 * source and the caller didn't indicate they would be sending more data after 869 * this. 870 */ 871 static void do_splice_eof(struct splice_desc *sd) 872 { 873 if (sd->splice_eof) 874 sd->splice_eof(sd); 875 } 876 877 /* 878 * Callers already called rw_verify_area() on the entire range. 879 * No need to call it for sub ranges. 880 */ 881 static ssize_t do_splice_read(struct file *in, loff_t *ppos, 882 struct pipe_inode_info *pipe, size_t len, 883 unsigned int flags) 884 { 885 unsigned int p_space; 886 887 if (unlikely(!(in->f_mode & FMODE_READ))) 888 return -EBADF; 889 if (!len) 890 return 0; 891 892 /* Don't try to read more the pipe has space for. */ 893 p_space = pipe->max_usage - pipe_buf_usage(pipe); 894 len = min_t(size_t, len, p_space << PAGE_SHIFT); 895 896 if (unlikely(len > MAX_RW_COUNT)) 897 len = MAX_RW_COUNT; 898 899 if (unlikely(!in->f_op->splice_read)) 900 return warn_unsupported(in, "read"); 901 /* 902 * O_DIRECT and DAX don't deal with the pagecache, so we allocate a 903 * buffer, copy into it and splice that into the pipe. 904 */ 905 if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host)) 906 return copy_splice_read(in, ppos, pipe, len, flags); 907 return in->f_op->splice_read(in, ppos, pipe, len, flags); 908 } 909 910 /** 911 * vfs_splice_read - Read data from a file and splice it into a pipe 912 * @in: File to splice from 913 * @ppos: Input file offset 914 * @pipe: Pipe to splice to 915 * @len: Number of bytes to splice 916 * @flags: Splice modifier flags (SPLICE_F_*) 917 * 918 * Splice the requested amount of data from the input file to the pipe. This 919 * is synchronous as the caller must hold the pipe lock across the entire 920 * operation. 921 * 922 * If successful, it returns the amount of data spliced, 0 if it hit the EOF or 923 * a hole and a negative error code otherwise. 924 */ 925 ssize_t vfs_splice_read(struct file *in, loff_t *ppos, 926 struct pipe_inode_info *pipe, size_t len, 927 unsigned int flags) 928 { 929 ssize_t ret; 930 931 ret = rw_verify_area(READ, in, ppos, len); 932 if (unlikely(ret < 0)) 933 return ret; 934 935 return do_splice_read(in, ppos, pipe, len, flags); 936 } 937 EXPORT_SYMBOL_GPL(vfs_splice_read); 938 939 /* 940 * This is a special case helper to splice directly between two 941 * points, without requiring an explicit pipe. Internally an allocated 942 * pipe is cached in the process, and reused during the lifetime of 943 * that process. 944 */ 945 static ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 946 splice_direct_actor *actor) 947 { 948 struct pipe_inode_info *pipe; 949 ssize_t ret, bytes; 950 size_t len; 951 int i, flags, more; 952 953 /* 954 * We require the input to be seekable, as we don't want to randomly 955 * drop data for eg socket -> socket splicing. Use the piped splicing 956 * for that! 957 */ 958 if (unlikely(!(in->f_mode & FMODE_LSEEK))) 959 return -EINVAL; 960 961 /* 962 * neither in nor out is a pipe, setup an internal pipe attached to 963 * 'out' and transfer the wanted data from 'in' to 'out' through that 964 */ 965 pipe = current->splice_pipe; 966 if (unlikely(!pipe)) { 967 pipe = alloc_pipe_info(); 968 if (!pipe) 969 return -ENOMEM; 970 971 /* 972 * We don't have an immediate reader, but we'll read the stuff 973 * out of the pipe right after the splice_to_pipe(). So set 974 * PIPE_READERS appropriately. 975 */ 976 pipe->readers = 1; 977 978 current->splice_pipe = pipe; 979 } 980 981 /* 982 * Do the splice. 983 */ 984 bytes = 0; 985 len = sd->total_len; 986 987 /* Don't block on output, we have to drain the direct pipe. */ 988 flags = sd->flags; 989 sd->flags &= ~SPLICE_F_NONBLOCK; 990 991 /* 992 * We signal MORE until we've read sufficient data to fulfill the 993 * request and we keep signalling it if the caller set it. 994 */ 995 more = sd->flags & SPLICE_F_MORE; 996 sd->flags |= SPLICE_F_MORE; 997 998 WARN_ON_ONCE(!pipe_is_empty(pipe)); 999 1000 while (len) { 1001 size_t read_len; 1002 loff_t pos = sd->pos, prev_pos = pos; 1003 1004 ret = do_splice_read(in, &pos, pipe, len, flags); 1005 if (unlikely(ret <= 0)) 1006 goto read_failure; 1007 1008 read_len = ret; 1009 sd->total_len = read_len; 1010 1011 /* 1012 * If we now have sufficient data to fulfill the request then 1013 * we clear SPLICE_F_MORE if it was not set initially. 1014 */ 1015 if (read_len >= len && !more) 1016 sd->flags &= ~SPLICE_F_MORE; 1017 1018 /* 1019 * NOTE: nonblocking mode only applies to the input. We 1020 * must not do the output in nonblocking mode as then we 1021 * could get stuck data in the internal pipe: 1022 */ 1023 ret = actor(pipe, sd); 1024 if (unlikely(ret <= 0)) { 1025 sd->pos = prev_pos; 1026 goto out_release; 1027 } 1028 1029 bytes += ret; 1030 len -= ret; 1031 sd->pos = pos; 1032 1033 if (ret < read_len) { 1034 sd->pos = prev_pos + ret; 1035 goto out_release; 1036 } 1037 } 1038 1039 done: 1040 pipe->tail = pipe->head = 0; 1041 file_accessed(in); 1042 return bytes; 1043 1044 read_failure: 1045 /* 1046 * If the user did *not* set SPLICE_F_MORE *and* we didn't hit that 1047 * "use all of len" case that cleared SPLICE_F_MORE, *and* we did a 1048 * "->splice_in()" that returned EOF (ie zero) *and* we have sent at 1049 * least 1 byte *then* we will also do the ->splice_eof() call. 1050 */ 1051 if (ret == 0 && !more && len > 0 && bytes) 1052 do_splice_eof(sd); 1053 out_release: 1054 /* 1055 * If we did an incomplete transfer we must release 1056 * the pipe buffers in question: 1057 */ 1058 for (i = 0; i < pipe->ring_size; i++) { 1059 struct pipe_buffer *buf = &pipe->bufs[i]; 1060 1061 if (buf->ops) 1062 pipe_buf_release(pipe, buf); 1063 } 1064 1065 if (!bytes) 1066 bytes = ret; 1067 1068 goto done; 1069 } 1070 1071 /** 1072 * vfs_splice_to_actor - call an actor on data read from a file 1073 * @in: file to read from 1074 * @pos: file offset 1075 * @count: maximum number of bytes to read 1076 * @actor: callback to process a pipe's worth of data 1077 * @private: private data passed to @actor 1078 * 1079 * Read up to @count worth of data from @in at @pos, and call @actor 1080 * when the hidden pipe used to buffer the data is full. Ensures the 1081 * read is allowed using rw_verify_area() and emits fsnotify access 1082 * events. @in must be seekable (FMODE_LSEEK). 1083 * 1084 * Return: The number of bytes spliced, or a negative errno. 1085 */ 1086 ssize_t vfs_splice_to_actor(struct file *in, loff_t pos, size_t count, 1087 splice_direct_actor *actor, void *private) 1088 { 1089 struct splice_desc sd = { 1090 .total_len = count, 1091 .pos = pos, 1092 .u.data = private, 1093 }; 1094 ssize_t ret; 1095 1096 ret = rw_verify_area(READ, in, &sd.pos, sd.total_len); 1097 if (ret < 0) 1098 return ret; 1099 1100 ret = splice_direct_to_actor(in, &sd, actor); 1101 if (ret >= 0) 1102 fsnotify_access(in); 1103 return ret; 1104 } 1105 EXPORT_SYMBOL(vfs_splice_to_actor); 1106 1107 static int direct_splice_actor(struct pipe_inode_info *pipe, 1108 struct splice_desc *sd) 1109 { 1110 struct file *file = sd->u.file; 1111 long ret; 1112 1113 file_start_write(file); 1114 ret = do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags); 1115 file_end_write(file); 1116 return ret; 1117 } 1118 1119 static int splice_file_range_actor(struct pipe_inode_info *pipe, 1120 struct splice_desc *sd) 1121 { 1122 struct file *file = sd->u.file; 1123 1124 return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags); 1125 } 1126 1127 static void direct_file_splice_eof(struct splice_desc *sd) 1128 { 1129 struct file *file = sd->u.file; 1130 1131 if (file->f_op->splice_eof) 1132 file->f_op->splice_eof(file); 1133 } 1134 1135 static ssize_t do_splice_direct_actor(struct file *in, loff_t *ppos, 1136 struct file *out, loff_t *opos, 1137 size_t len, unsigned int flags, 1138 splice_direct_actor *actor) 1139 { 1140 struct splice_desc sd = { 1141 .len = len, 1142 .total_len = len, 1143 .flags = flags, 1144 .pos = *ppos, 1145 .u.file = out, 1146 .splice_eof = direct_file_splice_eof, 1147 .opos = opos, 1148 }; 1149 ssize_t ret; 1150 1151 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1152 return -EBADF; 1153 1154 if (unlikely(out->f_flags & O_APPEND)) 1155 return -EINVAL; 1156 1157 ret = splice_direct_to_actor(in, &sd, actor); 1158 if (ret > 0) 1159 *ppos = sd.pos; 1160 1161 return ret; 1162 } 1163 /** 1164 * do_splice_direct - splices data directly between two files 1165 * @in: file to splice from 1166 * @ppos: input file offset 1167 * @out: file to splice to 1168 * @opos: output file offset 1169 * @len: number of bytes to splice 1170 * @flags: splice modifier flags 1171 * 1172 * Description: 1173 * For use by do_sendfile(). splice can easily emulate sendfile, but 1174 * doing it in the application would incur an extra system call 1175 * (splice in + splice out, as compared to just sendfile()). So this helper 1176 * can splice directly through a process-private pipe. 1177 * 1178 * Callers already called rw_verify_area() on the entire range. 1179 */ 1180 ssize_t do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1181 loff_t *opos, size_t len, unsigned int flags) 1182 { 1183 return do_splice_direct_actor(in, ppos, out, opos, len, flags, 1184 direct_splice_actor); 1185 } 1186 EXPORT_SYMBOL(do_splice_direct); 1187 1188 /** 1189 * splice_file_range - splices data between two files for copy_file_range() 1190 * @in: file to splice from 1191 * @ppos: input file offset 1192 * @out: file to splice to 1193 * @opos: output file offset 1194 * @len: number of bytes to splice 1195 * 1196 * Description: 1197 * For use by ->copy_file_range() methods. 1198 * Like do_splice_direct(), but vfs_copy_file_range() already holds 1199 * start_file_write() on @out file. 1200 * 1201 * Callers already called rw_verify_area() on the entire range. 1202 */ 1203 ssize_t splice_file_range(struct file *in, loff_t *ppos, struct file *out, 1204 loff_t *opos, size_t len) 1205 { 1206 lockdep_assert(file_write_started(out)); 1207 1208 return do_splice_direct_actor(in, ppos, out, opos, 1209 min_t(size_t, len, MAX_RW_COUNT), 1210 0, splice_file_range_actor); 1211 } 1212 EXPORT_SYMBOL(splice_file_range); 1213 1214 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) 1215 { 1216 for (;;) { 1217 if (unlikely(!pipe->readers)) { 1218 send_sig(SIGPIPE, current, 0); 1219 return -EPIPE; 1220 } 1221 if (!pipe_is_full(pipe)) 1222 return 0; 1223 if (flags & SPLICE_F_NONBLOCK) 1224 return -EAGAIN; 1225 if (signal_pending(current)) 1226 return -ERESTARTSYS; 1227 pipe_wait_writable(pipe); 1228 } 1229 } 1230 1231 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1232 struct pipe_inode_info *opipe, 1233 size_t len, unsigned int flags); 1234 1235 ssize_t splice_file_to_pipe(struct file *in, 1236 struct pipe_inode_info *opipe, 1237 loff_t *offset, 1238 size_t len, unsigned int flags) 1239 { 1240 ssize_t ret; 1241 1242 pipe_lock(opipe); 1243 ret = wait_for_space(opipe, flags); 1244 if (!ret) 1245 ret = do_splice_read(in, offset, opipe, len, flags); 1246 pipe_unlock(opipe); 1247 if (ret > 0) 1248 wakeup_pipe_readers(opipe); 1249 return ret; 1250 } 1251 1252 /* 1253 * Determine where to splice to/from. 1254 */ 1255 ssize_t do_splice(struct file *in, loff_t *off_in, struct file *out, 1256 loff_t *off_out, size_t len, unsigned int flags) 1257 { 1258 struct pipe_inode_info *ipipe; 1259 struct pipe_inode_info *opipe; 1260 loff_t offset; 1261 ssize_t ret; 1262 1263 if (unlikely(!(in->f_mode & FMODE_READ) || 1264 !(out->f_mode & FMODE_WRITE))) 1265 return -EBADF; 1266 1267 ipipe = get_pipe_info(in, true); 1268 opipe = get_pipe_info(out, true); 1269 1270 if (ipipe && opipe) { 1271 if (off_in || off_out) 1272 return -ESPIPE; 1273 1274 /* Splicing to self would be fun, but... */ 1275 if (ipipe == opipe) 1276 return -EINVAL; 1277 1278 if ((in->f_flags | out->f_flags) & O_NONBLOCK) 1279 flags |= SPLICE_F_NONBLOCK; 1280 1281 ret = splice_pipe_to_pipe(ipipe, opipe, len, flags); 1282 } else if (ipipe) { 1283 if (off_in) 1284 return -ESPIPE; 1285 if (off_out) { 1286 if (!(out->f_mode & FMODE_PWRITE)) 1287 return -EINVAL; 1288 offset = *off_out; 1289 } else { 1290 offset = out->f_pos; 1291 } 1292 1293 if (unlikely(out->f_flags & O_APPEND)) 1294 return -EINVAL; 1295 1296 ret = rw_verify_area(WRITE, out, &offset, len); 1297 if (unlikely(ret < 0)) 1298 return ret; 1299 1300 if (in->f_flags & O_NONBLOCK) 1301 flags |= SPLICE_F_NONBLOCK; 1302 1303 file_start_write(out); 1304 ret = do_splice_from(ipipe, out, &offset, len, flags); 1305 file_end_write(out); 1306 1307 if (off_out) 1308 *off_out = offset; 1309 else if (ret >= 0) 1310 out->f_pos = offset; 1311 } else if (opipe) { 1312 if (off_out) 1313 return -ESPIPE; 1314 if (off_in) { 1315 if (!(in->f_mode & FMODE_PREAD)) 1316 return -EINVAL; 1317 offset = *off_in; 1318 } else { 1319 offset = in->f_pos; 1320 } 1321 1322 ret = rw_verify_area(READ, in, &offset, len); 1323 if (unlikely(ret < 0)) 1324 return ret; 1325 1326 if (out->f_flags & O_NONBLOCK) 1327 flags |= SPLICE_F_NONBLOCK; 1328 1329 ret = splice_file_to_pipe(in, opipe, &offset, len, flags); 1330 1331 if (off_in) 1332 *off_in = offset; 1333 else if (ret >= 0) 1334 in->f_pos = offset; 1335 } else { 1336 ret = -EINVAL; 1337 } 1338 1339 if (ret > 0) { 1340 /* 1341 * Generate modify out before access in: 1342 * do_splice_from() may've already sent modify out, 1343 * and this ensures the events get merged. 1344 */ 1345 fsnotify_modify(out); 1346 fsnotify_access(in); 1347 } 1348 1349 return ret; 1350 } 1351 1352 static ssize_t __do_splice(struct file *in, loff_t __user *off_in, 1353 struct file *out, loff_t __user *off_out, 1354 size_t len, unsigned int flags) 1355 { 1356 struct pipe_inode_info *ipipe; 1357 struct pipe_inode_info *opipe; 1358 loff_t offset, *__off_in = NULL, *__off_out = NULL; 1359 ssize_t ret; 1360 1361 ipipe = get_pipe_info(in, true); 1362 opipe = get_pipe_info(out, true); 1363 1364 if (ipipe) { 1365 if (off_in) 1366 return -ESPIPE; 1367 pipe_clear_nowait(in); 1368 } 1369 if (opipe) { 1370 if (off_out) 1371 return -ESPIPE; 1372 pipe_clear_nowait(out); 1373 } 1374 1375 if (off_out) { 1376 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1377 return -EFAULT; 1378 __off_out = &offset; 1379 } 1380 if (off_in) { 1381 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1382 return -EFAULT; 1383 __off_in = &offset; 1384 } 1385 1386 ret = do_splice(in, __off_in, out, __off_out, len, flags); 1387 if (ret < 0) 1388 return ret; 1389 1390 if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t))) 1391 return -EFAULT; 1392 if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t))) 1393 return -EFAULT; 1394 1395 return ret; 1396 } 1397 1398 static ssize_t iter_to_pipe(struct iov_iter *from, 1399 struct pipe_inode_info *pipe) 1400 { 1401 struct pipe_buffer buf = { 1402 .ops = &user_page_pipe_buf_ops, 1403 }; 1404 size_t total = 0; 1405 ssize_t ret = 0; 1406 1407 while (iov_iter_count(from)) { 1408 struct page *pages[16]; 1409 ssize_t left; 1410 size_t start; 1411 int i, n; 1412 1413 left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start); 1414 if (left <= 0) { 1415 ret = left; 1416 break; 1417 } 1418 1419 n = DIV_ROUND_UP(left + start, PAGE_SIZE); 1420 for (i = 0; i < n; i++) { 1421 int size = umin(left, PAGE_SIZE - start); 1422 1423 buf.page = pages[i]; 1424 buf.offset = start; 1425 buf.len = size; 1426 ret = add_to_pipe(pipe, &buf); 1427 if (unlikely(ret < 0)) { 1428 iov_iter_revert(from, left); 1429 // this one got dropped by add_to_pipe() 1430 while (++i < n) 1431 put_page(pages[i]); 1432 goto out; 1433 } 1434 total += ret; 1435 left -= size; 1436 start = 0; 1437 } 1438 } 1439 out: 1440 return total ? total : ret; 1441 } 1442 1443 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1444 struct splice_desc *sd) 1445 { 1446 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); 1447 return n == sd->len ? n : -EFAULT; 1448 } 1449 1450 /* 1451 * For lack of a better implementation, implement vmsplice() to userspace 1452 * as a simple copy of the pipe's pages to the user iov. 1453 */ 1454 static ssize_t vmsplice_to_user(struct file *file, struct iov_iter *iter, 1455 unsigned int flags) 1456 { 1457 struct pipe_inode_info *pipe = get_pipe_info(file, true); 1458 struct splice_desc sd = { 1459 .total_len = iov_iter_count(iter), 1460 .flags = flags, 1461 .u.data = iter 1462 }; 1463 ssize_t ret = 0; 1464 1465 if (!pipe) 1466 return -EBADF; 1467 1468 pipe_clear_nowait(file); 1469 1470 if (sd.total_len) { 1471 pipe_lock(pipe); 1472 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); 1473 pipe_unlock(pipe); 1474 } 1475 1476 if (ret > 0) 1477 fsnotify_access(file); 1478 1479 return ret; 1480 } 1481 1482 /* 1483 * vmsplice splices a user address range into a pipe. It can be thought of 1484 * as splice-from-memory, where the regular splice is splice-from-file (or 1485 * to file). In both cases the output is a pipe, naturally. 1486 */ 1487 static ssize_t vmsplice_to_pipe(struct file *file, struct iov_iter *iter, 1488 unsigned int flags) 1489 { 1490 struct pipe_inode_info *pipe; 1491 ssize_t ret = 0; 1492 1493 pipe = get_pipe_info(file, true); 1494 if (!pipe) 1495 return -EBADF; 1496 1497 pipe_clear_nowait(file); 1498 1499 pipe_lock(pipe); 1500 ret = wait_for_space(pipe, flags); 1501 if (!ret) 1502 ret = iter_to_pipe(iter, pipe); 1503 pipe_unlock(pipe); 1504 if (ret > 0) { 1505 wakeup_pipe_readers(pipe); 1506 fsnotify_modify(file); 1507 } 1508 return ret; 1509 } 1510 1511 /* 1512 * Note that vmsplice only really supports true splicing _from_ user memory 1513 * to a pipe, not the other way around. Splicing from user memory is a simple 1514 * operation that can be supported without any funky alignment restrictions 1515 * or nasty vm tricks. We simply map in the user memory and fill them into 1516 * a pipe. The reverse isn't quite as easy, though. There are two possible 1517 * solutions for that: 1518 * 1519 * - memcpy() the data internally, at which point we might as well just 1520 * do a regular read() on the buffer anyway. 1521 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1522 * has restriction limitations on both ends of the pipe). 1523 * 1524 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1525 * 1526 */ 1527 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov, 1528 unsigned long, nr_segs, unsigned int, flags) 1529 { 1530 struct iovec iovstack[UIO_FASTIOV]; 1531 struct iovec *iov = iovstack; 1532 struct iov_iter iter; 1533 ssize_t error; 1534 int type; 1535 1536 if (unlikely(flags & ~SPLICE_F_ALL)) 1537 return -EINVAL; 1538 1539 CLASS(fd, f)(fd); 1540 if (fd_empty(f)) 1541 return -EBADF; 1542 if (fd_file(f)->f_mode & FMODE_WRITE) 1543 type = ITER_SOURCE; 1544 else if (fd_file(f)->f_mode & FMODE_READ) 1545 type = ITER_DEST; 1546 else 1547 return -EBADF; 1548 1549 error = import_iovec(type, uiov, nr_segs, 1550 ARRAY_SIZE(iovstack), &iov, &iter); 1551 if (error < 0) 1552 return error; 1553 1554 if (!iov_iter_count(&iter)) 1555 error = 0; 1556 else if (type == ITER_SOURCE) 1557 error = vmsplice_to_pipe(fd_file(f), &iter, flags); 1558 else 1559 error = vmsplice_to_user(fd_file(f), &iter, flags); 1560 1561 kfree(iov); 1562 return error; 1563 } 1564 1565 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1566 int, fd_out, loff_t __user *, off_out, 1567 size_t, len, unsigned int, flags) 1568 { 1569 if (unlikely(!len)) 1570 return 0; 1571 1572 if (unlikely(flags & ~SPLICE_F_ALL)) 1573 return -EINVAL; 1574 1575 CLASS(fd, in)(fd_in); 1576 if (fd_empty(in)) 1577 return -EBADF; 1578 1579 CLASS(fd, out)(fd_out); 1580 if (fd_empty(out)) 1581 return -EBADF; 1582 1583 return __do_splice(fd_file(in), off_in, fd_file(out), off_out, 1584 len, flags); 1585 } 1586 1587 /* 1588 * Make sure there's data to read. Wait for input if we can, otherwise 1589 * return an appropriate error. 1590 */ 1591 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1592 { 1593 int ret; 1594 1595 /* 1596 * Check the pipe occupancy without the inode lock first. This function 1597 * is speculative anyways, so missing one is ok. 1598 */ 1599 if (!pipe_is_empty(pipe)) 1600 return 0; 1601 1602 ret = 0; 1603 pipe_lock(pipe); 1604 1605 while (pipe_is_empty(pipe)) { 1606 if (signal_pending(current)) { 1607 ret = -ERESTARTSYS; 1608 break; 1609 } 1610 if (!pipe->writers) 1611 break; 1612 if (flags & SPLICE_F_NONBLOCK) { 1613 ret = -EAGAIN; 1614 break; 1615 } 1616 pipe_wait_readable(pipe); 1617 } 1618 1619 pipe_unlock(pipe); 1620 return ret; 1621 } 1622 1623 /* 1624 * Make sure there's writeable room. Wait for room if we can, otherwise 1625 * return an appropriate error. 1626 */ 1627 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1628 { 1629 int ret; 1630 1631 /* 1632 * Check pipe occupancy without the inode lock first. This function 1633 * is speculative anyways, so missing one is ok. 1634 */ 1635 if (!pipe_is_full(pipe)) 1636 return 0; 1637 1638 ret = 0; 1639 pipe_lock(pipe); 1640 1641 while (pipe_is_full(pipe)) { 1642 if (!pipe->readers) { 1643 send_sig(SIGPIPE, current, 0); 1644 ret = -EPIPE; 1645 break; 1646 } 1647 if (flags & SPLICE_F_NONBLOCK) { 1648 ret = -EAGAIN; 1649 break; 1650 } 1651 if (signal_pending(current)) { 1652 ret = -ERESTARTSYS; 1653 break; 1654 } 1655 pipe_wait_writable(pipe); 1656 } 1657 1658 pipe_unlock(pipe); 1659 return ret; 1660 } 1661 1662 /* 1663 * Splice contents of ipipe to opipe. 1664 */ 1665 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1666 struct pipe_inode_info *opipe, 1667 size_t len, unsigned int flags) 1668 { 1669 struct pipe_buffer *ibuf, *obuf; 1670 unsigned int i_head, o_head; 1671 unsigned int i_tail, o_tail; 1672 int ret = 0; 1673 bool input_wakeup = false; 1674 1675 1676 retry: 1677 ret = ipipe_prep(ipipe, flags); 1678 if (ret) 1679 return ret; 1680 1681 ret = opipe_prep(opipe, flags); 1682 if (ret) 1683 return ret; 1684 1685 /* 1686 * Potential ABBA deadlock, work around it by ordering lock 1687 * grabbing by pipe info address. Otherwise two different processes 1688 * could deadlock (one doing tee from A -> B, the other from B -> A). 1689 */ 1690 pipe_double_lock(ipipe, opipe); 1691 1692 i_tail = ipipe->tail; 1693 o_head = opipe->head; 1694 1695 do { 1696 size_t o_len; 1697 1698 if (!opipe->readers) { 1699 send_sig(SIGPIPE, current, 0); 1700 if (!ret) 1701 ret = -EPIPE; 1702 break; 1703 } 1704 1705 i_head = ipipe->head; 1706 o_tail = opipe->tail; 1707 1708 if (pipe_empty(i_head, i_tail) && !ipipe->writers) 1709 break; 1710 1711 /* 1712 * Cannot make any progress, because either the input 1713 * pipe is empty or the output pipe is full. 1714 */ 1715 if (pipe_empty(i_head, i_tail) || 1716 pipe_full(o_head, o_tail, opipe->max_usage)) { 1717 /* Already processed some buffers, break */ 1718 if (ret) 1719 break; 1720 1721 if (flags & SPLICE_F_NONBLOCK) { 1722 ret = -EAGAIN; 1723 break; 1724 } 1725 1726 /* 1727 * We raced with another reader/writer and haven't 1728 * managed to process any buffers. A zero return 1729 * value means EOF, so retry instead. 1730 */ 1731 pipe_unlock(ipipe); 1732 pipe_unlock(opipe); 1733 goto retry; 1734 } 1735 1736 ibuf = pipe_buf(ipipe, i_tail); 1737 obuf = pipe_buf(opipe, o_head); 1738 1739 if (len >= ibuf->len) { 1740 /* 1741 * Simply move the whole buffer from ipipe to opipe 1742 */ 1743 *obuf = *ibuf; 1744 ibuf->ops = NULL; 1745 i_tail++; 1746 ipipe->tail = i_tail; 1747 input_wakeup = true; 1748 o_len = obuf->len; 1749 o_head++; 1750 opipe->head = o_head; 1751 } else { 1752 /* 1753 * Get a reference to this pipe buffer, 1754 * so we can copy the contents over. 1755 */ 1756 if (!pipe_buf_get(ipipe, ibuf)) { 1757 if (ret == 0) 1758 ret = -EFAULT; 1759 break; 1760 } 1761 *obuf = *ibuf; 1762 1763 /* The page is shared with ipipe: don't append to it */ 1764 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; 1765 1766 obuf->len = len; 1767 ibuf->offset += len; 1768 ibuf->len -= len; 1769 o_len = len; 1770 o_head++; 1771 opipe->head = o_head; 1772 } 1773 ret += o_len; 1774 len -= o_len; 1775 } while (len); 1776 1777 pipe_unlock(ipipe); 1778 pipe_unlock(opipe); 1779 1780 /* 1781 * If we put data in the output pipe, wakeup any potential readers. 1782 */ 1783 if (ret > 0) 1784 wakeup_pipe_readers(opipe); 1785 1786 if (input_wakeup) 1787 wakeup_pipe_writers(ipipe); 1788 1789 return ret; 1790 } 1791 1792 /* 1793 * Link contents of ipipe to opipe. 1794 */ 1795 static ssize_t link_pipe(struct pipe_inode_info *ipipe, 1796 struct pipe_inode_info *opipe, 1797 size_t len, unsigned int flags) 1798 { 1799 struct pipe_buffer *ibuf, *obuf; 1800 unsigned int i_head, o_head; 1801 unsigned int i_tail, o_tail; 1802 ssize_t ret = 0; 1803 1804 retry: 1805 ret = ipipe_prep(ipipe, flags); 1806 if (ret) 1807 return ret; 1808 1809 ret = opipe_prep(opipe, flags); 1810 if (ret) 1811 return ret; 1812 1813 /* 1814 * Potential ABBA deadlock, work around it by ordering lock 1815 * grabbing by pipe info address. Otherwise two different processes 1816 * could deadlock (one doing tee from A -> B, the other from B -> A). 1817 */ 1818 pipe_double_lock(ipipe, opipe); 1819 1820 i_tail = ipipe->tail; 1821 o_head = opipe->head; 1822 1823 do { 1824 if (!opipe->readers) { 1825 send_sig(SIGPIPE, current, 0); 1826 if (!ret) 1827 ret = -EPIPE; 1828 break; 1829 } 1830 1831 i_head = ipipe->head; 1832 o_tail = opipe->tail; 1833 1834 if (pipe_empty(i_head, i_tail) && !ipipe->writers) 1835 break; 1836 1837 /* 1838 * If we have iterated all input buffers or run out of 1839 * output room, break. 1840 */ 1841 if (pipe_empty(i_head, i_tail) || 1842 pipe_full(o_head, o_tail, opipe->max_usage)) { 1843 if (ret) 1844 break; 1845 1846 if (flags & SPLICE_F_NONBLOCK) { 1847 ret = -EAGAIN; 1848 break; 1849 } 1850 1851 /* 1852 * We raced with another reader/writer and haven't 1853 * linked any buffers. A zero return value means EOF, 1854 * so retry instead. 1855 */ 1856 pipe_unlock(ipipe); 1857 pipe_unlock(opipe); 1858 goto retry; 1859 } 1860 1861 ibuf = pipe_buf(ipipe, i_tail); 1862 obuf = pipe_buf(opipe, o_head); 1863 1864 /* 1865 * Get a reference to this pipe buffer, 1866 * so we can copy the contents over. 1867 */ 1868 if (!pipe_buf_get(ipipe, ibuf)) { 1869 if (ret == 0) 1870 ret = -EFAULT; 1871 break; 1872 } 1873 1874 *obuf = *ibuf; 1875 1876 /* The page is shared with ipipe: don't append to it */ 1877 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; 1878 1879 if (obuf->len > len) 1880 obuf->len = len; 1881 ret += obuf->len; 1882 len -= obuf->len; 1883 1884 o_head++; 1885 opipe->head = o_head; 1886 i_tail++; 1887 } while (len); 1888 1889 pipe_unlock(ipipe); 1890 pipe_unlock(opipe); 1891 1892 /* 1893 * If we put data in the output pipe, wakeup any potential readers. 1894 */ 1895 if (ret > 0) 1896 wakeup_pipe_readers(opipe); 1897 1898 return ret; 1899 } 1900 1901 /* 1902 * This is a tee(1) implementation that works on pipes. It doesn't copy 1903 * any data, it simply references the 'in' pages on the 'out' pipe. 1904 * The 'flags' used are the SPLICE_F_* variants, currently the only 1905 * applicable one is SPLICE_F_NONBLOCK. 1906 */ 1907 ssize_t do_tee(struct file *in, struct file *out, size_t len, 1908 unsigned int flags) 1909 { 1910 struct pipe_inode_info *ipipe = get_pipe_info(in, true); 1911 struct pipe_inode_info *opipe = get_pipe_info(out, true); 1912 ssize_t ret = -EINVAL; 1913 1914 if (unlikely(!(in->f_mode & FMODE_READ) || 1915 !(out->f_mode & FMODE_WRITE))) 1916 return -EBADF; 1917 1918 /* 1919 * Duplicate the contents of ipipe to opipe without actually 1920 * copying the data. 1921 */ 1922 if (ipipe && opipe && ipipe != opipe) { 1923 if ((in->f_flags | out->f_flags) & O_NONBLOCK) 1924 flags |= SPLICE_F_NONBLOCK; 1925 1926 ret = link_pipe(ipipe, opipe, len, flags); 1927 } 1928 1929 if (ret > 0) { 1930 fsnotify_access(in); 1931 fsnotify_modify(out); 1932 } 1933 1934 return ret; 1935 } 1936 1937 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 1938 { 1939 if (unlikely(flags & ~SPLICE_F_ALL)) 1940 return -EINVAL; 1941 1942 if (unlikely(!len)) 1943 return 0; 1944 1945 CLASS(fd, in)(fdin); 1946 if (fd_empty(in)) 1947 return -EBADF; 1948 1949 CLASS(fd, out)(fdout); 1950 if (fd_empty(out)) 1951 return -EBADF; 1952 1953 return do_tee(fd_file(in), fd_file(out), len, flags); 1954 } 1955