1 /*- 2 * Copyright (c) 1996 John S. Dyson 3 * Copyright (c) 2012 Giovanni Trematerra 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice immediately at the beginning of the file, without modification, 11 * this list of conditions, and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Absolutely no warranty of function or purpose is made by the author 16 * John S. Dyson. 17 * 4. Modifications may be freely made to this file if the above conditions 18 * are met. 19 */ 20 21 /* 22 * This file contains a high-performance replacement for the socket-based 23 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 24 * all features of sockets, but does do everything that pipes normally 25 * do. 26 */ 27 28 /* 29 * This code has two modes of operation, a small write mode and a large 30 * write mode. The small write mode acts like conventional pipes with 31 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 32 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 33 * and PIPE_SIZE in size, the sending process pins the underlying pages in 34 * memory, and the receiving process copies directly from these pinned pages 35 * in the sending process. 36 * 37 * If the sending process receives a signal, it is possible that it will 38 * go away, and certainly its address space can change, because control 39 * is returned back to the user-mode side. In that case, the pipe code 40 * arranges to copy the buffer supplied by the user process, to a pageable 41 * kernel buffer, and the receiving process will grab the data from the 42 * pageable kernel buffer. Since signals don't happen all that often, 43 * the copy operation is normally eliminated. 44 * 45 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 46 * happen for small transfers so that the system will not spend all of 47 * its time context switching. 48 * 49 * In order to limit the resource use of pipes, two sysctls exist: 50 * 51 * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable 52 * address space available to us in pipe_map. This value is normally 53 * autotuned, but may also be loader tuned. 54 * 55 * kern.ipc.pipekva - This read-only sysctl tracks the current amount of 56 * memory in use by pipes. 57 * 58 * Based on how large pipekva is relative to maxpipekva, the following 59 * will happen: 60 * 61 * 0% - 50%: 62 * New pipes are given 16K of memory backing, pipes may dynamically 63 * grow to as large as 64K where needed. 64 * 50% - 75%: 65 * New pipes are given 4K (or PAGE_SIZE) of memory backing, 66 * existing pipes may NOT grow. 67 * 75% - 100%: 68 * New pipes are given 4K (or PAGE_SIZE) of memory backing, 69 * existing pipes will be shrunk down to 4K whenever possible. 70 * 71 * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0. If 72 * that is set, the only resize that will occur is the 0 -> SMALL_PIPE_SIZE 73 * resize which MUST occur for reverse-direction pipes when they are 74 * first used. 75 * 76 * Additional information about the current state of pipes may be obtained 77 * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail, 78 * and kern.ipc.piperesizefail. 79 * 80 * Locking rules: There are two locks present here: A mutex, used via 81 * PIPE_LOCK, and a flag, used via pipelock(). All locking is done via 82 * the flag, as mutexes can not persist over uiomove. The mutex 83 * exists only to guard access to the flag, and is not in itself a 84 * locking mechanism. Also note that there is only a single mutex for 85 * both directions of a pipe. 86 * 87 * As pipelock() may have to sleep before it can acquire the flag, it 88 * is important to reread all data after a call to pipelock(); everything 89 * in the structure may have changed. 90 */ 91 92 #include <sys/cdefs.h> 93 __FBSDID("$FreeBSD$"); 94 95 #include <sys/param.h> 96 #include <sys/systm.h> 97 #include <sys/conf.h> 98 #include <sys/fcntl.h> 99 #include <sys/file.h> 100 #include <sys/filedesc.h> 101 #include <sys/filio.h> 102 #include <sys/kernel.h> 103 #include <sys/lock.h> 104 #include <sys/mutex.h> 105 #include <sys/ttycom.h> 106 #include <sys/stat.h> 107 #include <sys/malloc.h> 108 #include <sys/poll.h> 109 #include <sys/selinfo.h> 110 #include <sys/signalvar.h> 111 #include <sys/syscallsubr.h> 112 #include <sys/sysctl.h> 113 #include <sys/sysproto.h> 114 #include <sys/pipe.h> 115 #include <sys/proc.h> 116 #include <sys/vnode.h> 117 #include <sys/uio.h> 118 #include <sys/event.h> 119 120 #include <security/mac/mac_framework.h> 121 122 #include <vm/vm.h> 123 #include <vm/vm_param.h> 124 #include <vm/vm_object.h> 125 #include <vm/vm_kern.h> 126 #include <vm/vm_extern.h> 127 #include <vm/pmap.h> 128 #include <vm/vm_map.h> 129 #include <vm/vm_page.h> 130 #include <vm/uma.h> 131 132 #include <fs/fifofs/fifo.h> 133 134 /* 135 * Use this define if you want to disable *fancy* VM things. Expect an 136 * approx 30% decrease in transfer rate. This could be useful for 137 * NetBSD or OpenBSD. 138 */ 139 /* #define PIPE_NODIRECT */ 140 141 #define PIPE_PEER(pipe) \ 142 (((pipe)->pipe_state & PIPE_NAMED) ? (pipe) : ((pipe)->pipe_peer)) 143 144 /* 145 * interfaces to the outside world 146 */ 147 static fo_rdwr_t pipe_read; 148 static fo_rdwr_t pipe_write; 149 static fo_truncate_t pipe_truncate; 150 static fo_ioctl_t pipe_ioctl; 151 static fo_poll_t pipe_poll; 152 static fo_kqfilter_t pipe_kqfilter; 153 static fo_stat_t pipe_stat; 154 static fo_close_t pipe_close; 155 static fo_chmod_t pipe_chmod; 156 static fo_chown_t pipe_chown; 157 158 struct fileops pipeops = { 159 .fo_read = pipe_read, 160 .fo_write = pipe_write, 161 .fo_truncate = pipe_truncate, 162 .fo_ioctl = pipe_ioctl, 163 .fo_poll = pipe_poll, 164 .fo_kqfilter = pipe_kqfilter, 165 .fo_stat = pipe_stat, 166 .fo_close = pipe_close, 167 .fo_chmod = pipe_chmod, 168 .fo_chown = pipe_chown, 169 .fo_flags = DFLAG_PASSABLE 170 }; 171 172 static void filt_pipedetach(struct knote *kn); 173 static void filt_pipedetach_notsup(struct knote *kn); 174 static int filt_pipenotsup(struct knote *kn, long hint); 175 static int filt_piperead(struct knote *kn, long hint); 176 static int filt_pipewrite(struct knote *kn, long hint); 177 178 static struct filterops pipe_nfiltops = { 179 .f_isfd = 1, 180 .f_detach = filt_pipedetach_notsup, 181 .f_event = filt_pipenotsup 182 }; 183 static struct filterops pipe_rfiltops = { 184 .f_isfd = 1, 185 .f_detach = filt_pipedetach, 186 .f_event = filt_piperead 187 }; 188 static struct filterops pipe_wfiltops = { 189 .f_isfd = 1, 190 .f_detach = filt_pipedetach, 191 .f_event = filt_pipewrite 192 }; 193 194 /* 195 * Default pipe buffer size(s), this can be kind-of large now because pipe 196 * space is pageable. The pipe code will try to maintain locality of 197 * reference for performance reasons, so small amounts of outstanding I/O 198 * will not wipe the cache. 199 */ 200 #define MINPIPESIZE (PIPE_SIZE/3) 201 #define MAXPIPESIZE (2*PIPE_SIZE/3) 202 203 static long amountpipekva; 204 static int pipefragretry; 205 static int pipeallocfail; 206 static int piperesizefail; 207 static int piperesizeallowed = 1; 208 209 SYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN, 210 &maxpipekva, 0, "Pipe KVA limit"); 211 SYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 212 &amountpipekva, 0, "Pipe KVA usage"); 213 SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD, 214 &pipefragretry, 0, "Pipe allocation retries due to fragmentation"); 215 SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD, 216 &pipeallocfail, 0, "Pipe allocation failures"); 217 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD, 218 &piperesizefail, 0, "Pipe resize failures"); 219 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW, 220 &piperesizeallowed, 0, "Pipe resizing allowed"); 221 222 static void pipeinit(void *dummy __unused); 223 static void pipeclose(struct pipe *cpipe); 224 static void pipe_free_kmem(struct pipe *cpipe); 225 static int pipe_create(struct pipe *pipe, int backing); 226 static int pipe_paircreate(struct thread *td, struct pipepair **p_pp); 227 static __inline int pipelock(struct pipe *cpipe, int catch); 228 static __inline void pipeunlock(struct pipe *cpipe); 229 static __inline void pipeselwakeup(struct pipe *cpipe); 230 #ifndef PIPE_NODIRECT 231 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 232 static void pipe_destroy_write_buffer(struct pipe *wpipe); 233 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 234 static void pipe_clone_write_buffer(struct pipe *wpipe); 235 #endif 236 static int pipespace(struct pipe *cpipe, int size); 237 static int pipespace_new(struct pipe *cpipe, int size); 238 239 static int pipe_zone_ctor(void *mem, int size, void *arg, int flags); 240 static int pipe_zone_init(void *mem, int size, int flags); 241 static void pipe_zone_fini(void *mem, int size); 242 243 static uma_zone_t pipe_zone; 244 static struct unrhdr *pipeino_unr; 245 static dev_t pipedev_ino; 246 247 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 248 249 static void 250 pipeinit(void *dummy __unused) 251 { 252 253 pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair), 254 pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini, 255 UMA_ALIGN_PTR, 0); 256 KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 257 pipeino_unr = new_unrhdr(1, INT32_MAX, NULL); 258 KASSERT(pipeino_unr != NULL, ("pipe fake inodes not initialized")); 259 pipedev_ino = devfs_alloc_cdp_inode(); 260 KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized")); 261 } 262 263 static int 264 pipe_zone_ctor(void *mem, int size, void *arg, int flags) 265 { 266 struct pipepair *pp; 267 struct pipe *rpipe, *wpipe; 268 269 KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size")); 270 271 pp = (struct pipepair *)mem; 272 273 /* 274 * We zero both pipe endpoints to make sure all the kmem pointers 275 * are NULL, flag fields are zero'd, etc. We timestamp both 276 * endpoints with the same time. 277 */ 278 rpipe = &pp->pp_rpipe; 279 bzero(rpipe, sizeof(*rpipe)); 280 vfs_timestamp(&rpipe->pipe_ctime); 281 rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime; 282 283 wpipe = &pp->pp_wpipe; 284 bzero(wpipe, sizeof(*wpipe)); 285 wpipe->pipe_ctime = rpipe->pipe_ctime; 286 wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime; 287 288 rpipe->pipe_peer = wpipe; 289 rpipe->pipe_pair = pp; 290 wpipe->pipe_peer = rpipe; 291 wpipe->pipe_pair = pp; 292 293 /* 294 * Mark both endpoints as present; they will later get free'd 295 * one at a time. When both are free'd, then the whole pair 296 * is released. 297 */ 298 rpipe->pipe_present = PIPE_ACTIVE; 299 wpipe->pipe_present = PIPE_ACTIVE; 300 301 /* 302 * Eventually, the MAC Framework may initialize the label 303 * in ctor or init, but for now we do it elswhere to avoid 304 * blocking in ctor or init. 305 */ 306 pp->pp_label = NULL; 307 308 return (0); 309 } 310 311 static int 312 pipe_zone_init(void *mem, int size, int flags) 313 { 314 struct pipepair *pp; 315 316 KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size")); 317 318 pp = (struct pipepair *)mem; 319 320 mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 321 return (0); 322 } 323 324 static void 325 pipe_zone_fini(void *mem, int size) 326 { 327 struct pipepair *pp; 328 329 KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size")); 330 331 pp = (struct pipepair *)mem; 332 333 mtx_destroy(&pp->pp_mtx); 334 } 335 336 static int 337 pipe_paircreate(struct thread *td, struct pipepair **p_pp) 338 { 339 struct pipepair *pp; 340 struct pipe *rpipe, *wpipe; 341 int error; 342 343 *p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK); 344 #ifdef MAC 345 /* 346 * The MAC label is shared between the connected endpoints. As a 347 * result mac_pipe_init() and mac_pipe_create() are called once 348 * for the pair, and not on the endpoints. 349 */ 350 mac_pipe_init(pp); 351 mac_pipe_create(td->td_ucred, pp); 352 #endif 353 rpipe = &pp->pp_rpipe; 354 wpipe = &pp->pp_wpipe; 355 356 knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe)); 357 knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe)); 358 359 /* Only the forward direction pipe is backed by default */ 360 if ((error = pipe_create(rpipe, 1)) != 0 || 361 (error = pipe_create(wpipe, 0)) != 0) { 362 pipeclose(rpipe); 363 pipeclose(wpipe); 364 return (error); 365 } 366 367 rpipe->pipe_state |= PIPE_DIRECTOK; 368 wpipe->pipe_state |= PIPE_DIRECTOK; 369 return (0); 370 } 371 372 int 373 pipe_named_ctor(struct pipe **ppipe, struct thread *td) 374 { 375 struct pipepair *pp; 376 int error; 377 378 error = pipe_paircreate(td, &pp); 379 if (error != 0) 380 return (error); 381 pp->pp_rpipe.pipe_state |= PIPE_NAMED; 382 *ppipe = &pp->pp_rpipe; 383 return (0); 384 } 385 386 void 387 pipe_dtor(struct pipe *dpipe) 388 { 389 ino_t ino; 390 391 ino = dpipe->pipe_ino; 392 funsetown(&dpipe->pipe_sigio); 393 pipeclose(dpipe); 394 if (dpipe->pipe_state & PIPE_NAMED) { 395 dpipe = dpipe->pipe_peer; 396 funsetown(&dpipe->pipe_sigio); 397 pipeclose(dpipe); 398 } 399 if (ino != 0 && ino != (ino_t)-1) 400 free_unr(pipeino_unr, ino); 401 } 402 403 /* 404 * The pipe system call for the DTYPE_PIPE type of pipes. If we fail, let 405 * the zone pick up the pieces via pipeclose(). 406 */ 407 int 408 kern_pipe(struct thread *td, int fildes[2]) 409 { 410 struct filedesc *fdp; 411 struct file *rf, *wf; 412 struct pipe *rpipe, *wpipe; 413 struct pipepair *pp; 414 int fd, error; 415 416 fdp = td->td_proc->p_fd; 417 error = pipe_paircreate(td, &pp); 418 if (error != 0) 419 return (error); 420 rpipe = &pp->pp_rpipe; 421 wpipe = &pp->pp_wpipe; 422 error = falloc(td, &rf, &fd, 0); 423 if (error) { 424 pipeclose(rpipe); 425 pipeclose(wpipe); 426 return (error); 427 } 428 /* An extra reference on `rf' has been held for us by falloc(). */ 429 fildes[0] = fd; 430 431 /* 432 * Warning: once we've gotten past allocation of the fd for the 433 * read-side, we can only drop the read side via fdrop() in order 434 * to avoid races against processes which manage to dup() the read 435 * side while we are blocked trying to allocate the write side. 436 */ 437 finit(rf, FREAD | FWRITE, DTYPE_PIPE, rpipe, &pipeops); 438 error = falloc(td, &wf, &fd, 0); 439 if (error) { 440 fdclose(fdp, rf, fildes[0], td); 441 fdrop(rf, td); 442 /* rpipe has been closed by fdrop(). */ 443 pipeclose(wpipe); 444 return (error); 445 } 446 /* An extra reference on `wf' has been held for us by falloc(). */ 447 finit(wf, FREAD | FWRITE, DTYPE_PIPE, wpipe, &pipeops); 448 fdrop(wf, td); 449 fildes[1] = fd; 450 fdrop(rf, td); 451 452 return (0); 453 } 454 455 /* ARGSUSED */ 456 int 457 sys_pipe(struct thread *td, struct pipe_args *uap) 458 { 459 int error; 460 int fildes[2]; 461 462 error = kern_pipe(td, fildes); 463 if (error) 464 return (error); 465 466 td->td_retval[0] = fildes[0]; 467 td->td_retval[1] = fildes[1]; 468 469 return (0); 470 } 471 472 /* 473 * Allocate kva for pipe circular buffer, the space is pageable 474 * This routine will 'realloc' the size of a pipe safely, if it fails 475 * it will retain the old buffer. 476 * If it fails it will return ENOMEM. 477 */ 478 static int 479 pipespace_new(cpipe, size) 480 struct pipe *cpipe; 481 int size; 482 { 483 caddr_t buffer; 484 int error, cnt, firstseg; 485 static int curfail = 0; 486 static struct timeval lastfail; 487 488 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked")); 489 KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW), 490 ("pipespace: resize of direct writes not allowed")); 491 retry: 492 cnt = cpipe->pipe_buffer.cnt; 493 if (cnt > size) 494 size = cnt; 495 496 size = round_page(size); 497 buffer = (caddr_t) vm_map_min(pipe_map); 498 499 error = vm_map_find(pipe_map, NULL, 0, 500 (vm_offset_t *) &buffer, size, 1, 501 VM_PROT_ALL, VM_PROT_ALL, 0); 502 if (error != KERN_SUCCESS) { 503 if ((cpipe->pipe_buffer.buffer == NULL) && 504 (size > SMALL_PIPE_SIZE)) { 505 size = SMALL_PIPE_SIZE; 506 pipefragretry++; 507 goto retry; 508 } 509 if (cpipe->pipe_buffer.buffer == NULL) { 510 pipeallocfail++; 511 if (ppsratecheck(&lastfail, &curfail, 1)) 512 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n"); 513 } else { 514 piperesizefail++; 515 } 516 return (ENOMEM); 517 } 518 519 /* copy data, then free old resources if we're resizing */ 520 if (cnt > 0) { 521 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) { 522 firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out; 523 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 524 buffer, firstseg); 525 if ((cnt - firstseg) > 0) 526 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg], 527 cpipe->pipe_buffer.in); 528 } else { 529 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 530 buffer, cnt); 531 } 532 } 533 pipe_free_kmem(cpipe); 534 cpipe->pipe_buffer.buffer = buffer; 535 cpipe->pipe_buffer.size = size; 536 cpipe->pipe_buffer.in = cnt; 537 cpipe->pipe_buffer.out = 0; 538 cpipe->pipe_buffer.cnt = cnt; 539 atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size); 540 return (0); 541 } 542 543 /* 544 * Wrapper for pipespace_new() that performs locking assertions. 545 */ 546 static int 547 pipespace(cpipe, size) 548 struct pipe *cpipe; 549 int size; 550 { 551 552 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 553 ("Unlocked pipe passed to pipespace")); 554 return (pipespace_new(cpipe, size)); 555 } 556 557 /* 558 * lock a pipe for I/O, blocking other access 559 */ 560 static __inline int 561 pipelock(cpipe, catch) 562 struct pipe *cpipe; 563 int catch; 564 { 565 int error; 566 567 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 568 while (cpipe->pipe_state & PIPE_LOCKFL) { 569 cpipe->pipe_state |= PIPE_LWANT; 570 error = msleep(cpipe, PIPE_MTX(cpipe), 571 catch ? (PRIBIO | PCATCH) : PRIBIO, 572 "pipelk", 0); 573 if (error != 0) 574 return (error); 575 } 576 cpipe->pipe_state |= PIPE_LOCKFL; 577 return (0); 578 } 579 580 /* 581 * unlock a pipe I/O lock 582 */ 583 static __inline void 584 pipeunlock(cpipe) 585 struct pipe *cpipe; 586 { 587 588 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 589 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 590 ("Unlocked pipe passed to pipeunlock")); 591 cpipe->pipe_state &= ~PIPE_LOCKFL; 592 if (cpipe->pipe_state & PIPE_LWANT) { 593 cpipe->pipe_state &= ~PIPE_LWANT; 594 wakeup(cpipe); 595 } 596 } 597 598 static __inline void 599 pipeselwakeup(cpipe) 600 struct pipe *cpipe; 601 { 602 603 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 604 if (cpipe->pipe_state & PIPE_SEL) { 605 selwakeuppri(&cpipe->pipe_sel, PSOCK); 606 if (!SEL_WAITING(&cpipe->pipe_sel)) 607 cpipe->pipe_state &= ~PIPE_SEL; 608 } 609 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 610 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 611 KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0); 612 } 613 614 /* 615 * Initialize and allocate VM and memory for pipe. The structure 616 * will start out zero'd from the ctor, so we just manage the kmem. 617 */ 618 static int 619 pipe_create(pipe, backing) 620 struct pipe *pipe; 621 int backing; 622 { 623 int error; 624 625 if (backing) { 626 if (amountpipekva > maxpipekva / 2) 627 error = pipespace_new(pipe, SMALL_PIPE_SIZE); 628 else 629 error = pipespace_new(pipe, PIPE_SIZE); 630 } else { 631 /* If we're not backing this pipe, no need to do anything. */ 632 error = 0; 633 } 634 pipe->pipe_ino = -1; 635 return (error); 636 } 637 638 /* ARGSUSED */ 639 static int 640 pipe_read(fp, uio, active_cred, flags, td) 641 struct file *fp; 642 struct uio *uio; 643 struct ucred *active_cred; 644 struct thread *td; 645 int flags; 646 { 647 struct pipe *rpipe; 648 int error; 649 int nread = 0; 650 u_int size; 651 652 rpipe = fp->f_data; 653 PIPE_LOCK(rpipe); 654 ++rpipe->pipe_busy; 655 error = pipelock(rpipe, 1); 656 if (error) 657 goto unlocked_error; 658 659 #ifdef MAC 660 error = mac_pipe_check_read(active_cred, rpipe->pipe_pair); 661 if (error) 662 goto locked_error; 663 #endif 664 if (amountpipekva > (3 * maxpipekva) / 4) { 665 if (!(rpipe->pipe_state & PIPE_DIRECTW) && 666 (rpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 667 (rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 668 (piperesizeallowed == 1)) { 669 PIPE_UNLOCK(rpipe); 670 pipespace(rpipe, SMALL_PIPE_SIZE); 671 PIPE_LOCK(rpipe); 672 } 673 } 674 675 while (uio->uio_resid) { 676 /* 677 * normal pipe buffer receive 678 */ 679 if (rpipe->pipe_buffer.cnt > 0) { 680 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 681 if (size > rpipe->pipe_buffer.cnt) 682 size = rpipe->pipe_buffer.cnt; 683 if (size > uio->uio_resid) 684 size = (u_int) uio->uio_resid; 685 686 PIPE_UNLOCK(rpipe); 687 error = uiomove( 688 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 689 size, uio); 690 PIPE_LOCK(rpipe); 691 if (error) 692 break; 693 694 rpipe->pipe_buffer.out += size; 695 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 696 rpipe->pipe_buffer.out = 0; 697 698 rpipe->pipe_buffer.cnt -= size; 699 700 /* 701 * If there is no more to read in the pipe, reset 702 * its pointers to the beginning. This improves 703 * cache hit stats. 704 */ 705 if (rpipe->pipe_buffer.cnt == 0) { 706 rpipe->pipe_buffer.in = 0; 707 rpipe->pipe_buffer.out = 0; 708 } 709 nread += size; 710 #ifndef PIPE_NODIRECT 711 /* 712 * Direct copy, bypassing a kernel buffer. 713 */ 714 } else if ((size = rpipe->pipe_map.cnt) && 715 (rpipe->pipe_state & PIPE_DIRECTW)) { 716 if (size > uio->uio_resid) 717 size = (u_int) uio->uio_resid; 718 719 PIPE_UNLOCK(rpipe); 720 error = uiomove_fromphys(rpipe->pipe_map.ms, 721 rpipe->pipe_map.pos, size, uio); 722 PIPE_LOCK(rpipe); 723 if (error) 724 break; 725 nread += size; 726 rpipe->pipe_map.pos += size; 727 rpipe->pipe_map.cnt -= size; 728 if (rpipe->pipe_map.cnt == 0) { 729 rpipe->pipe_state &= ~PIPE_DIRECTW; 730 wakeup(rpipe); 731 } 732 #endif 733 } else { 734 /* 735 * detect EOF condition 736 * read returns 0 on EOF, no need to set error 737 */ 738 if (rpipe->pipe_state & PIPE_EOF) 739 break; 740 741 /* 742 * If the "write-side" has been blocked, wake it up now. 743 */ 744 if (rpipe->pipe_state & PIPE_WANTW) { 745 rpipe->pipe_state &= ~PIPE_WANTW; 746 wakeup(rpipe); 747 } 748 749 /* 750 * Break if some data was read. 751 */ 752 if (nread > 0) 753 break; 754 755 /* 756 * Unlock the pipe buffer for our remaining processing. 757 * We will either break out with an error or we will 758 * sleep and relock to loop. 759 */ 760 pipeunlock(rpipe); 761 762 /* 763 * Handle non-blocking mode operation or 764 * wait for more data. 765 */ 766 if (fp->f_flag & FNONBLOCK) { 767 error = EAGAIN; 768 } else { 769 rpipe->pipe_state |= PIPE_WANTR; 770 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 771 PRIBIO | PCATCH, 772 "piperd", 0)) == 0) 773 error = pipelock(rpipe, 1); 774 } 775 if (error) 776 goto unlocked_error; 777 } 778 } 779 #ifdef MAC 780 locked_error: 781 #endif 782 pipeunlock(rpipe); 783 784 /* XXX: should probably do this before getting any locks. */ 785 if (error == 0) 786 vfs_timestamp(&rpipe->pipe_atime); 787 unlocked_error: 788 --rpipe->pipe_busy; 789 790 /* 791 * PIPE_WANT processing only makes sense if pipe_busy is 0. 792 */ 793 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 794 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 795 wakeup(rpipe); 796 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 797 /* 798 * Handle write blocking hysteresis. 799 */ 800 if (rpipe->pipe_state & PIPE_WANTW) { 801 rpipe->pipe_state &= ~PIPE_WANTW; 802 wakeup(rpipe); 803 } 804 } 805 806 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 807 pipeselwakeup(rpipe); 808 809 PIPE_UNLOCK(rpipe); 810 return (error); 811 } 812 813 #ifndef PIPE_NODIRECT 814 /* 815 * Map the sending processes' buffer into kernel space and wire it. 816 * This is similar to a physical write operation. 817 */ 818 static int 819 pipe_build_write_buffer(wpipe, uio) 820 struct pipe *wpipe; 821 struct uio *uio; 822 { 823 u_int size; 824 int i; 825 826 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 827 KASSERT(wpipe->pipe_state & PIPE_DIRECTW, 828 ("Clone attempt on non-direct write pipe!")); 829 830 if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size) 831 size = wpipe->pipe_buffer.size; 832 else 833 size = uio->uio_iov->iov_len; 834 835 if ((i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 836 (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ, 837 wpipe->pipe_map.ms, PIPENPAGES)) < 0) 838 return (EFAULT); 839 840 /* 841 * set up the control block 842 */ 843 wpipe->pipe_map.npages = i; 844 wpipe->pipe_map.pos = 845 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 846 wpipe->pipe_map.cnt = size; 847 848 /* 849 * and update the uio data 850 */ 851 852 uio->uio_iov->iov_len -= size; 853 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 854 if (uio->uio_iov->iov_len == 0) 855 uio->uio_iov++; 856 uio->uio_resid -= size; 857 uio->uio_offset += size; 858 return (0); 859 } 860 861 /* 862 * unmap and unwire the process buffer 863 */ 864 static void 865 pipe_destroy_write_buffer(wpipe) 866 struct pipe *wpipe; 867 { 868 869 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 870 vm_page_unhold_pages(wpipe->pipe_map.ms, wpipe->pipe_map.npages); 871 wpipe->pipe_map.npages = 0; 872 } 873 874 /* 875 * In the case of a signal, the writing process might go away. This 876 * code copies the data into the circular buffer so that the source 877 * pages can be freed without loss of data. 878 */ 879 static void 880 pipe_clone_write_buffer(wpipe) 881 struct pipe *wpipe; 882 { 883 struct uio uio; 884 struct iovec iov; 885 int size; 886 int pos; 887 888 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 889 size = wpipe->pipe_map.cnt; 890 pos = wpipe->pipe_map.pos; 891 892 wpipe->pipe_buffer.in = size; 893 wpipe->pipe_buffer.out = 0; 894 wpipe->pipe_buffer.cnt = size; 895 wpipe->pipe_state &= ~PIPE_DIRECTW; 896 897 PIPE_UNLOCK(wpipe); 898 iov.iov_base = wpipe->pipe_buffer.buffer; 899 iov.iov_len = size; 900 uio.uio_iov = &iov; 901 uio.uio_iovcnt = 1; 902 uio.uio_offset = 0; 903 uio.uio_resid = size; 904 uio.uio_segflg = UIO_SYSSPACE; 905 uio.uio_rw = UIO_READ; 906 uio.uio_td = curthread; 907 uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio); 908 PIPE_LOCK(wpipe); 909 pipe_destroy_write_buffer(wpipe); 910 } 911 912 /* 913 * This implements the pipe buffer write mechanism. Note that only 914 * a direct write OR a normal pipe write can be pending at any given time. 915 * If there are any characters in the pipe buffer, the direct write will 916 * be deferred until the receiving process grabs all of the bytes from 917 * the pipe buffer. Then the direct mapping write is set-up. 918 */ 919 static int 920 pipe_direct_write(wpipe, uio) 921 struct pipe *wpipe; 922 struct uio *uio; 923 { 924 int error; 925 926 retry: 927 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 928 error = pipelock(wpipe, 1); 929 if (wpipe->pipe_state & PIPE_EOF) 930 error = EPIPE; 931 if (error) { 932 pipeunlock(wpipe); 933 goto error1; 934 } 935 while (wpipe->pipe_state & PIPE_DIRECTW) { 936 if (wpipe->pipe_state & PIPE_WANTR) { 937 wpipe->pipe_state &= ~PIPE_WANTR; 938 wakeup(wpipe); 939 } 940 pipeselwakeup(wpipe); 941 wpipe->pipe_state |= PIPE_WANTW; 942 pipeunlock(wpipe); 943 error = msleep(wpipe, PIPE_MTX(wpipe), 944 PRIBIO | PCATCH, "pipdww", 0); 945 if (error) 946 goto error1; 947 else 948 goto retry; 949 } 950 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 951 if (wpipe->pipe_buffer.cnt > 0) { 952 if (wpipe->pipe_state & PIPE_WANTR) { 953 wpipe->pipe_state &= ~PIPE_WANTR; 954 wakeup(wpipe); 955 } 956 pipeselwakeup(wpipe); 957 wpipe->pipe_state |= PIPE_WANTW; 958 pipeunlock(wpipe); 959 error = msleep(wpipe, PIPE_MTX(wpipe), 960 PRIBIO | PCATCH, "pipdwc", 0); 961 if (error) 962 goto error1; 963 else 964 goto retry; 965 } 966 967 wpipe->pipe_state |= PIPE_DIRECTW; 968 969 PIPE_UNLOCK(wpipe); 970 error = pipe_build_write_buffer(wpipe, uio); 971 PIPE_LOCK(wpipe); 972 if (error) { 973 wpipe->pipe_state &= ~PIPE_DIRECTW; 974 pipeunlock(wpipe); 975 goto error1; 976 } 977 978 error = 0; 979 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 980 if (wpipe->pipe_state & PIPE_EOF) { 981 pipe_destroy_write_buffer(wpipe); 982 pipeselwakeup(wpipe); 983 pipeunlock(wpipe); 984 error = EPIPE; 985 goto error1; 986 } 987 if (wpipe->pipe_state & PIPE_WANTR) { 988 wpipe->pipe_state &= ~PIPE_WANTR; 989 wakeup(wpipe); 990 } 991 pipeselwakeup(wpipe); 992 pipeunlock(wpipe); 993 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 994 "pipdwt", 0); 995 pipelock(wpipe, 0); 996 } 997 998 if (wpipe->pipe_state & PIPE_EOF) 999 error = EPIPE; 1000 if (wpipe->pipe_state & PIPE_DIRECTW) { 1001 /* 1002 * this bit of trickery substitutes a kernel buffer for 1003 * the process that might be going away. 1004 */ 1005 pipe_clone_write_buffer(wpipe); 1006 } else { 1007 pipe_destroy_write_buffer(wpipe); 1008 } 1009 pipeunlock(wpipe); 1010 return (error); 1011 1012 error1: 1013 wakeup(wpipe); 1014 return (error); 1015 } 1016 #endif 1017 1018 static int 1019 pipe_write(fp, uio, active_cred, flags, td) 1020 struct file *fp; 1021 struct uio *uio; 1022 struct ucred *active_cred; 1023 struct thread *td; 1024 int flags; 1025 { 1026 int error; 1027 size_t desiredsize, orig_resid; 1028 struct pipe *wpipe, *rpipe; 1029 1030 rpipe = fp->f_data; 1031 wpipe = PIPE_PEER(rpipe); 1032 PIPE_LOCK(rpipe); 1033 error = pipelock(wpipe, 1); 1034 if (error) { 1035 PIPE_UNLOCK(rpipe); 1036 return (error); 1037 } 1038 /* 1039 * detect loss of pipe read side, issue SIGPIPE if lost. 1040 */ 1041 if (wpipe->pipe_present != PIPE_ACTIVE || 1042 (wpipe->pipe_state & PIPE_EOF)) { 1043 pipeunlock(wpipe); 1044 PIPE_UNLOCK(rpipe); 1045 return (EPIPE); 1046 } 1047 #ifdef MAC 1048 error = mac_pipe_check_write(active_cred, wpipe->pipe_pair); 1049 if (error) { 1050 pipeunlock(wpipe); 1051 PIPE_UNLOCK(rpipe); 1052 return (error); 1053 } 1054 #endif 1055 ++wpipe->pipe_busy; 1056 1057 /* Choose a larger size if it's advantageous */ 1058 desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size); 1059 while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) { 1060 if (piperesizeallowed != 1) 1061 break; 1062 if (amountpipekva > maxpipekva / 2) 1063 break; 1064 if (desiredsize == BIG_PIPE_SIZE) 1065 break; 1066 desiredsize = desiredsize * 2; 1067 } 1068 1069 /* Choose a smaller size if we're in a OOM situation */ 1070 if ((amountpipekva > (3 * maxpipekva) / 4) && 1071 (wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 1072 (wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 1073 (piperesizeallowed == 1)) 1074 desiredsize = SMALL_PIPE_SIZE; 1075 1076 /* Resize if the above determined that a new size was necessary */ 1077 if ((desiredsize != wpipe->pipe_buffer.size) && 1078 ((wpipe->pipe_state & PIPE_DIRECTW) == 0)) { 1079 PIPE_UNLOCK(wpipe); 1080 pipespace(wpipe, desiredsize); 1081 PIPE_LOCK(wpipe); 1082 } 1083 if (wpipe->pipe_buffer.size == 0) { 1084 /* 1085 * This can only happen for reverse direction use of pipes 1086 * in a complete OOM situation. 1087 */ 1088 error = ENOMEM; 1089 --wpipe->pipe_busy; 1090 pipeunlock(wpipe); 1091 PIPE_UNLOCK(wpipe); 1092 return (error); 1093 } 1094 1095 pipeunlock(wpipe); 1096 1097 orig_resid = uio->uio_resid; 1098 1099 while (uio->uio_resid) { 1100 int space; 1101 1102 pipelock(wpipe, 0); 1103 if (wpipe->pipe_state & PIPE_EOF) { 1104 pipeunlock(wpipe); 1105 error = EPIPE; 1106 break; 1107 } 1108 #ifndef PIPE_NODIRECT 1109 /* 1110 * If the transfer is large, we can gain performance if 1111 * we do process-to-process copies directly. 1112 * If the write is non-blocking, we don't use the 1113 * direct write mechanism. 1114 * 1115 * The direct write mechanism will detect the reader going 1116 * away on us. 1117 */ 1118 if (uio->uio_segflg == UIO_USERSPACE && 1119 uio->uio_iov->iov_len >= PIPE_MINDIRECT && 1120 wpipe->pipe_buffer.size >= PIPE_MINDIRECT && 1121 (fp->f_flag & FNONBLOCK) == 0) { 1122 pipeunlock(wpipe); 1123 error = pipe_direct_write(wpipe, uio); 1124 if (error) 1125 break; 1126 continue; 1127 } 1128 #endif 1129 1130 /* 1131 * Pipe buffered writes cannot be coincidental with 1132 * direct writes. We wait until the currently executing 1133 * direct write is completed before we start filling the 1134 * pipe buffer. We break out if a signal occurs or the 1135 * reader goes away. 1136 */ 1137 if (wpipe->pipe_state & PIPE_DIRECTW) { 1138 if (wpipe->pipe_state & PIPE_WANTR) { 1139 wpipe->pipe_state &= ~PIPE_WANTR; 1140 wakeup(wpipe); 1141 } 1142 pipeselwakeup(wpipe); 1143 wpipe->pipe_state |= PIPE_WANTW; 1144 pipeunlock(wpipe); 1145 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1146 "pipbww", 0); 1147 if (error) 1148 break; 1149 else 1150 continue; 1151 } 1152 1153 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1154 1155 /* Writes of size <= PIPE_BUF must be atomic. */ 1156 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1157 space = 0; 1158 1159 if (space > 0) { 1160 int size; /* Transfer size */ 1161 int segsize; /* first segment to transfer */ 1162 1163 /* 1164 * Transfer size is minimum of uio transfer 1165 * and free space in pipe buffer. 1166 */ 1167 if (space > uio->uio_resid) 1168 size = uio->uio_resid; 1169 else 1170 size = space; 1171 /* 1172 * First segment to transfer is minimum of 1173 * transfer size and contiguous space in 1174 * pipe buffer. If first segment to transfer 1175 * is less than the transfer size, we've got 1176 * a wraparound in the buffer. 1177 */ 1178 segsize = wpipe->pipe_buffer.size - 1179 wpipe->pipe_buffer.in; 1180 if (segsize > size) 1181 segsize = size; 1182 1183 /* Transfer first segment */ 1184 1185 PIPE_UNLOCK(rpipe); 1186 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1187 segsize, uio); 1188 PIPE_LOCK(rpipe); 1189 1190 if (error == 0 && segsize < size) { 1191 KASSERT(wpipe->pipe_buffer.in + segsize == 1192 wpipe->pipe_buffer.size, 1193 ("Pipe buffer wraparound disappeared")); 1194 /* 1195 * Transfer remaining part now, to 1196 * support atomic writes. Wraparound 1197 * happened. 1198 */ 1199 1200 PIPE_UNLOCK(rpipe); 1201 error = uiomove( 1202 &wpipe->pipe_buffer.buffer[0], 1203 size - segsize, uio); 1204 PIPE_LOCK(rpipe); 1205 } 1206 if (error == 0) { 1207 wpipe->pipe_buffer.in += size; 1208 if (wpipe->pipe_buffer.in >= 1209 wpipe->pipe_buffer.size) { 1210 KASSERT(wpipe->pipe_buffer.in == 1211 size - segsize + 1212 wpipe->pipe_buffer.size, 1213 ("Expected wraparound bad")); 1214 wpipe->pipe_buffer.in = size - segsize; 1215 } 1216 1217 wpipe->pipe_buffer.cnt += size; 1218 KASSERT(wpipe->pipe_buffer.cnt <= 1219 wpipe->pipe_buffer.size, 1220 ("Pipe buffer overflow")); 1221 } 1222 pipeunlock(wpipe); 1223 if (error != 0) 1224 break; 1225 } else { 1226 /* 1227 * If the "read-side" has been blocked, wake it up now. 1228 */ 1229 if (wpipe->pipe_state & PIPE_WANTR) { 1230 wpipe->pipe_state &= ~PIPE_WANTR; 1231 wakeup(wpipe); 1232 } 1233 1234 /* 1235 * don't block on non-blocking I/O 1236 */ 1237 if (fp->f_flag & FNONBLOCK) { 1238 error = EAGAIN; 1239 pipeunlock(wpipe); 1240 break; 1241 } 1242 1243 /* 1244 * We have no more space and have something to offer, 1245 * wake up select/poll. 1246 */ 1247 pipeselwakeup(wpipe); 1248 1249 wpipe->pipe_state |= PIPE_WANTW; 1250 pipeunlock(wpipe); 1251 error = msleep(wpipe, PIPE_MTX(rpipe), 1252 PRIBIO | PCATCH, "pipewr", 0); 1253 if (error != 0) 1254 break; 1255 } 1256 } 1257 1258 pipelock(wpipe, 0); 1259 --wpipe->pipe_busy; 1260 1261 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1262 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1263 wakeup(wpipe); 1264 } else if (wpipe->pipe_buffer.cnt > 0) { 1265 /* 1266 * If we have put any characters in the buffer, we wake up 1267 * the reader. 1268 */ 1269 if (wpipe->pipe_state & PIPE_WANTR) { 1270 wpipe->pipe_state &= ~PIPE_WANTR; 1271 wakeup(wpipe); 1272 } 1273 } 1274 1275 /* 1276 * Don't return EPIPE if I/O was successful 1277 */ 1278 if ((wpipe->pipe_buffer.cnt == 0) && 1279 (uio->uio_resid == 0) && 1280 (error == EPIPE)) { 1281 error = 0; 1282 } 1283 1284 if (error == 0) 1285 vfs_timestamp(&wpipe->pipe_mtime); 1286 1287 /* 1288 * We have something to offer, 1289 * wake up select/poll. 1290 */ 1291 if (wpipe->pipe_buffer.cnt) 1292 pipeselwakeup(wpipe); 1293 1294 pipeunlock(wpipe); 1295 PIPE_UNLOCK(rpipe); 1296 return (error); 1297 } 1298 1299 /* ARGSUSED */ 1300 static int 1301 pipe_truncate(fp, length, active_cred, td) 1302 struct file *fp; 1303 off_t length; 1304 struct ucred *active_cred; 1305 struct thread *td; 1306 { 1307 1308 /* For named pipes call the vnode operation. */ 1309 if (fp->f_vnode != NULL) 1310 return (vnops.fo_truncate(fp, length, active_cred, td)); 1311 return (EINVAL); 1312 } 1313 1314 /* 1315 * we implement a very minimal set of ioctls for compatibility with sockets. 1316 */ 1317 static int 1318 pipe_ioctl(fp, cmd, data, active_cred, td) 1319 struct file *fp; 1320 u_long cmd; 1321 void *data; 1322 struct ucred *active_cred; 1323 struct thread *td; 1324 { 1325 struct pipe *mpipe = fp->f_data; 1326 int error; 1327 1328 PIPE_LOCK(mpipe); 1329 1330 #ifdef MAC 1331 error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data); 1332 if (error) { 1333 PIPE_UNLOCK(mpipe); 1334 return (error); 1335 } 1336 #endif 1337 1338 error = 0; 1339 switch (cmd) { 1340 1341 case FIONBIO: 1342 break; 1343 1344 case FIOASYNC: 1345 if (*(int *)data) { 1346 mpipe->pipe_state |= PIPE_ASYNC; 1347 } else { 1348 mpipe->pipe_state &= ~PIPE_ASYNC; 1349 } 1350 break; 1351 1352 case FIONREAD: 1353 if (!(fp->f_flag & FREAD)) { 1354 *(int *)data = 0; 1355 PIPE_UNLOCK(mpipe); 1356 return (0); 1357 } 1358 if (mpipe->pipe_state & PIPE_DIRECTW) 1359 *(int *)data = mpipe->pipe_map.cnt; 1360 else 1361 *(int *)data = mpipe->pipe_buffer.cnt; 1362 break; 1363 1364 case FIOSETOWN: 1365 PIPE_UNLOCK(mpipe); 1366 error = fsetown(*(int *)data, &mpipe->pipe_sigio); 1367 goto out_unlocked; 1368 1369 case FIOGETOWN: 1370 *(int *)data = fgetown(&mpipe->pipe_sigio); 1371 break; 1372 1373 /* This is deprecated, FIOSETOWN should be used instead. */ 1374 case TIOCSPGRP: 1375 PIPE_UNLOCK(mpipe); 1376 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio); 1377 goto out_unlocked; 1378 1379 /* This is deprecated, FIOGETOWN should be used instead. */ 1380 case TIOCGPGRP: 1381 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1382 break; 1383 1384 default: 1385 error = ENOTTY; 1386 break; 1387 } 1388 PIPE_UNLOCK(mpipe); 1389 out_unlocked: 1390 return (error); 1391 } 1392 1393 static int 1394 pipe_poll(fp, events, active_cred, td) 1395 struct file *fp; 1396 int events; 1397 struct ucred *active_cred; 1398 struct thread *td; 1399 { 1400 struct pipe *rpipe; 1401 struct pipe *wpipe; 1402 int levents, revents; 1403 #ifdef MAC 1404 int error; 1405 #endif 1406 1407 revents = 0; 1408 rpipe = fp->f_data; 1409 wpipe = PIPE_PEER(rpipe); 1410 PIPE_LOCK(rpipe); 1411 #ifdef MAC 1412 error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair); 1413 if (error) 1414 goto locked_error; 1415 #endif 1416 if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) 1417 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1418 (rpipe->pipe_buffer.cnt > 0)) 1419 revents |= events & (POLLIN | POLLRDNORM); 1420 1421 if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) 1422 if (wpipe->pipe_present != PIPE_ACTIVE || 1423 (wpipe->pipe_state & PIPE_EOF) || 1424 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1425 ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF || 1426 wpipe->pipe_buffer.size == 0))) 1427 revents |= events & (POLLOUT | POLLWRNORM); 1428 1429 levents = events & 1430 (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND); 1431 if (rpipe->pipe_state & PIPE_NAMED && fp->f_flag & FREAD && levents && 1432 fifo_iseof(fp)) 1433 events |= POLLINIGNEOF; 1434 1435 if ((events & POLLINIGNEOF) == 0) { 1436 if (rpipe->pipe_state & PIPE_EOF) { 1437 revents |= (events & (POLLIN | POLLRDNORM)); 1438 if (wpipe->pipe_present != PIPE_ACTIVE || 1439 (wpipe->pipe_state & PIPE_EOF)) 1440 revents |= POLLHUP; 1441 } 1442 } 1443 1444 if (revents == 0) { 1445 if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) { 1446 selrecord(td, &rpipe->pipe_sel); 1447 if (SEL_WAITING(&rpipe->pipe_sel)) 1448 rpipe->pipe_state |= PIPE_SEL; 1449 } 1450 1451 if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) { 1452 selrecord(td, &wpipe->pipe_sel); 1453 if (SEL_WAITING(&wpipe->pipe_sel)) 1454 wpipe->pipe_state |= PIPE_SEL; 1455 } 1456 } 1457 #ifdef MAC 1458 locked_error: 1459 #endif 1460 PIPE_UNLOCK(rpipe); 1461 1462 return (revents); 1463 } 1464 1465 /* 1466 * We shouldn't need locks here as we're doing a read and this should 1467 * be a natural race. 1468 */ 1469 static int 1470 pipe_stat(fp, ub, active_cred, td) 1471 struct file *fp; 1472 struct stat *ub; 1473 struct ucred *active_cred; 1474 struct thread *td; 1475 { 1476 struct pipe *pipe; 1477 int new_unr; 1478 #ifdef MAC 1479 int error; 1480 #endif 1481 1482 pipe = fp->f_data; 1483 PIPE_LOCK(pipe); 1484 #ifdef MAC 1485 error = mac_pipe_check_stat(active_cred, pipe->pipe_pair); 1486 if (error) { 1487 PIPE_UNLOCK(pipe); 1488 return (error); 1489 } 1490 #endif 1491 1492 /* For named pipes ask the underlying filesystem. */ 1493 if (pipe->pipe_state & PIPE_NAMED) { 1494 PIPE_UNLOCK(pipe); 1495 return (vnops.fo_stat(fp, ub, active_cred, td)); 1496 } 1497 1498 /* 1499 * Lazily allocate an inode number for the pipe. Most pipe 1500 * users do not call fstat(2) on the pipe, which means that 1501 * postponing the inode allocation until it is must be 1502 * returned to userland is useful. If alloc_unr failed, 1503 * assign st_ino zero instead of returning an error. 1504 * Special pipe_ino values: 1505 * -1 - not yet initialized; 1506 * 0 - alloc_unr failed, return 0 as st_ino forever. 1507 */ 1508 if (pipe->pipe_ino == (ino_t)-1) { 1509 new_unr = alloc_unr(pipeino_unr); 1510 if (new_unr != -1) 1511 pipe->pipe_ino = new_unr; 1512 else 1513 pipe->pipe_ino = 0; 1514 } 1515 PIPE_UNLOCK(pipe); 1516 1517 bzero(ub, sizeof(*ub)); 1518 ub->st_mode = S_IFIFO; 1519 ub->st_blksize = PAGE_SIZE; 1520 if (pipe->pipe_state & PIPE_DIRECTW) 1521 ub->st_size = pipe->pipe_map.cnt; 1522 else 1523 ub->st_size = pipe->pipe_buffer.cnt; 1524 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1525 ub->st_atim = pipe->pipe_atime; 1526 ub->st_mtim = pipe->pipe_mtime; 1527 ub->st_ctim = pipe->pipe_ctime; 1528 ub->st_uid = fp->f_cred->cr_uid; 1529 ub->st_gid = fp->f_cred->cr_gid; 1530 ub->st_dev = pipedev_ino; 1531 ub->st_ino = pipe->pipe_ino; 1532 /* 1533 * Left as 0: st_nlink, st_rdev, st_flags, st_gen. 1534 */ 1535 return (0); 1536 } 1537 1538 /* ARGSUSED */ 1539 static int 1540 pipe_close(fp, td) 1541 struct file *fp; 1542 struct thread *td; 1543 { 1544 1545 if (fp->f_vnode != NULL) 1546 return vnops.fo_close(fp, td); 1547 fp->f_ops = &badfileops; 1548 pipe_dtor(fp->f_data); 1549 fp->f_data = NULL; 1550 return (0); 1551 } 1552 1553 static int 1554 pipe_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td) 1555 { 1556 struct pipe *cpipe; 1557 int error; 1558 1559 cpipe = fp->f_data; 1560 if (cpipe->pipe_state & PIPE_NAMED) 1561 error = vn_chmod(fp, mode, active_cred, td); 1562 else 1563 error = invfo_chmod(fp, mode, active_cred, td); 1564 return (error); 1565 } 1566 1567 static int 1568 pipe_chown(fp, uid, gid, active_cred, td) 1569 struct file *fp; 1570 uid_t uid; 1571 gid_t gid; 1572 struct ucred *active_cred; 1573 struct thread *td; 1574 { 1575 struct pipe *cpipe; 1576 int error; 1577 1578 cpipe = fp->f_data; 1579 if (cpipe->pipe_state & PIPE_NAMED) 1580 error = vn_chown(fp, uid, gid, active_cred, td); 1581 else 1582 error = invfo_chown(fp, uid, gid, active_cred, td); 1583 return (error); 1584 } 1585 1586 static void 1587 pipe_free_kmem(cpipe) 1588 struct pipe *cpipe; 1589 { 1590 1591 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), 1592 ("pipe_free_kmem: pipe mutex locked")); 1593 1594 if (cpipe->pipe_buffer.buffer != NULL) { 1595 atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size); 1596 vm_map_remove(pipe_map, 1597 (vm_offset_t)cpipe->pipe_buffer.buffer, 1598 (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size); 1599 cpipe->pipe_buffer.buffer = NULL; 1600 } 1601 #ifndef PIPE_NODIRECT 1602 { 1603 cpipe->pipe_map.cnt = 0; 1604 cpipe->pipe_map.pos = 0; 1605 cpipe->pipe_map.npages = 0; 1606 } 1607 #endif 1608 } 1609 1610 /* 1611 * shutdown the pipe 1612 */ 1613 static void 1614 pipeclose(cpipe) 1615 struct pipe *cpipe; 1616 { 1617 struct pipepair *pp; 1618 struct pipe *ppipe; 1619 1620 KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL")); 1621 1622 PIPE_LOCK(cpipe); 1623 pipelock(cpipe, 0); 1624 pp = cpipe->pipe_pair; 1625 1626 pipeselwakeup(cpipe); 1627 1628 /* 1629 * If the other side is blocked, wake it up saying that 1630 * we want to close it down. 1631 */ 1632 cpipe->pipe_state |= PIPE_EOF; 1633 while (cpipe->pipe_busy) { 1634 wakeup(cpipe); 1635 cpipe->pipe_state |= PIPE_WANT; 1636 pipeunlock(cpipe); 1637 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1638 pipelock(cpipe, 0); 1639 } 1640 1641 1642 /* 1643 * Disconnect from peer, if any. 1644 */ 1645 ppipe = cpipe->pipe_peer; 1646 if (ppipe->pipe_present == PIPE_ACTIVE) { 1647 pipeselwakeup(ppipe); 1648 1649 ppipe->pipe_state |= PIPE_EOF; 1650 wakeup(ppipe); 1651 KNOTE_LOCKED(&ppipe->pipe_sel.si_note, 0); 1652 } 1653 1654 /* 1655 * Mark this endpoint as free. Release kmem resources. We 1656 * don't mark this endpoint as unused until we've finished 1657 * doing that, or the pipe might disappear out from under 1658 * us. 1659 */ 1660 PIPE_UNLOCK(cpipe); 1661 pipe_free_kmem(cpipe); 1662 PIPE_LOCK(cpipe); 1663 cpipe->pipe_present = PIPE_CLOSING; 1664 pipeunlock(cpipe); 1665 1666 /* 1667 * knlist_clear() may sleep dropping the PIPE_MTX. Set the 1668 * PIPE_FINALIZED, that allows other end to free the 1669 * pipe_pair, only after the knotes are completely dismantled. 1670 */ 1671 knlist_clear(&cpipe->pipe_sel.si_note, 1); 1672 cpipe->pipe_present = PIPE_FINALIZED; 1673 seldrain(&cpipe->pipe_sel); 1674 knlist_destroy(&cpipe->pipe_sel.si_note); 1675 1676 /* 1677 * If both endpoints are now closed, release the memory for the 1678 * pipe pair. If not, unlock. 1679 */ 1680 if (ppipe->pipe_present == PIPE_FINALIZED) { 1681 PIPE_UNLOCK(cpipe); 1682 #ifdef MAC 1683 mac_pipe_destroy(pp); 1684 #endif 1685 uma_zfree(pipe_zone, cpipe->pipe_pair); 1686 } else 1687 PIPE_UNLOCK(cpipe); 1688 } 1689 1690 /*ARGSUSED*/ 1691 static int 1692 pipe_kqfilter(struct file *fp, struct knote *kn) 1693 { 1694 struct pipe *cpipe; 1695 1696 /* 1697 * If a filter is requested that is not supported by this file 1698 * descriptor, don't return an error, but also don't ever generate an 1699 * event. 1700 */ 1701 if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) { 1702 kn->kn_fop = &pipe_nfiltops; 1703 return (0); 1704 } 1705 if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) { 1706 kn->kn_fop = &pipe_nfiltops; 1707 return (0); 1708 } 1709 cpipe = fp->f_data; 1710 PIPE_LOCK(cpipe); 1711 switch (kn->kn_filter) { 1712 case EVFILT_READ: 1713 kn->kn_fop = &pipe_rfiltops; 1714 break; 1715 case EVFILT_WRITE: 1716 kn->kn_fop = &pipe_wfiltops; 1717 if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) { 1718 /* other end of pipe has been closed */ 1719 PIPE_UNLOCK(cpipe); 1720 return (EPIPE); 1721 } 1722 cpipe = PIPE_PEER(cpipe); 1723 break; 1724 default: 1725 PIPE_UNLOCK(cpipe); 1726 return (EINVAL); 1727 } 1728 1729 kn->kn_hook = cpipe; 1730 knlist_add(&cpipe->pipe_sel.si_note, kn, 1); 1731 PIPE_UNLOCK(cpipe); 1732 return (0); 1733 } 1734 1735 static void 1736 filt_pipedetach(struct knote *kn) 1737 { 1738 struct pipe *cpipe = kn->kn_hook; 1739 1740 PIPE_LOCK(cpipe); 1741 knlist_remove(&cpipe->pipe_sel.si_note, kn, 1); 1742 PIPE_UNLOCK(cpipe); 1743 } 1744 1745 /*ARGSUSED*/ 1746 static int 1747 filt_piperead(struct knote *kn, long hint) 1748 { 1749 struct pipe *rpipe = kn->kn_hook; 1750 struct pipe *wpipe = rpipe->pipe_peer; 1751 int ret; 1752 1753 PIPE_LOCK(rpipe); 1754 kn->kn_data = rpipe->pipe_buffer.cnt; 1755 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1756 kn->kn_data = rpipe->pipe_map.cnt; 1757 1758 if ((rpipe->pipe_state & PIPE_EOF) || 1759 wpipe->pipe_present != PIPE_ACTIVE || 1760 (wpipe->pipe_state & PIPE_EOF)) { 1761 kn->kn_flags |= EV_EOF; 1762 PIPE_UNLOCK(rpipe); 1763 return (1); 1764 } 1765 ret = kn->kn_data > 0; 1766 PIPE_UNLOCK(rpipe); 1767 return ret; 1768 } 1769 1770 /*ARGSUSED*/ 1771 static int 1772 filt_pipewrite(struct knote *kn, long hint) 1773 { 1774 struct pipe *wpipe; 1775 1776 wpipe = kn->kn_hook; 1777 PIPE_LOCK(wpipe); 1778 if (wpipe->pipe_present != PIPE_ACTIVE || 1779 (wpipe->pipe_state & PIPE_EOF)) { 1780 kn->kn_data = 0; 1781 kn->kn_flags |= EV_EOF; 1782 PIPE_UNLOCK(wpipe); 1783 return (1); 1784 } 1785 kn->kn_data = (wpipe->pipe_buffer.size > 0) ? 1786 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) : PIPE_BUF; 1787 if (wpipe->pipe_state & PIPE_DIRECTW) 1788 kn->kn_data = 0; 1789 1790 PIPE_UNLOCK(wpipe); 1791 return (kn->kn_data >= PIPE_BUF); 1792 } 1793 1794 static void 1795 filt_pipedetach_notsup(struct knote *kn) 1796 { 1797 1798 } 1799 1800 static int 1801 filt_pipenotsup(struct knote *kn, long hint) 1802 { 1803 1804 return (0); 1805 } 1806