1 /*- 2 * Copyright (c) 1996 John S. Dyson 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice immediately at the beginning of the file, without modification, 10 * this list of conditions, and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. Absolutely no warranty of function or purpose is made by the author 15 * John S. Dyson. 16 * 4. Modifications may be freely made to this file if the above conditions 17 * are met. 18 */ 19 20 /* 21 * This file contains a high-performance replacement for the socket-based 22 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support 23 * all features of sockets, but does do everything that pipes normally 24 * do. 25 */ 26 27 /* 28 * This code has two modes of operation, a small write mode and a large 29 * write mode. The small write mode acts like conventional pipes with 30 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the 31 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT 32 * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and 33 * the receiving process can copy it directly from the pages in the sending 34 * process. 35 * 36 * If the sending process receives a signal, it is possible that it will 37 * go away, and certainly its address space can change, because control 38 * is returned back to the user-mode side. In that case, the pipe code 39 * arranges to copy the buffer supplied by the user process, to a pageable 40 * kernel buffer, and the receiving process will grab the data from the 41 * pageable kernel buffer. Since signals don't happen all that often, 42 * the copy operation is normally eliminated. 43 * 44 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will 45 * happen for small transfers so that the system will not spend all of 46 * its time context switching. 47 * 48 * In order to limit the resource use of pipes, two sysctls exist: 49 * 50 * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable 51 * address space available to us in pipe_map. This value is normally 52 * autotuned, but may also be loader tuned. 53 * 54 * kern.ipc.pipekva - This read-only sysctl tracks the current amount of 55 * memory in use by pipes. 56 * 57 * Based on how large pipekva is relative to maxpipekva, the following 58 * will happen: 59 * 60 * 0% - 50%: 61 * New pipes are given 16K of memory backing, pipes may dynamically 62 * grow to as large as 64K where needed. 63 * 50% - 75%: 64 * New pipes are given 4K (or PAGE_SIZE) of memory backing, 65 * existing pipes may NOT grow. 66 * 75% - 100%: 67 * New pipes are given 4K (or PAGE_SIZE) of memory backing, 68 * existing pipes will be shrunk down to 4K whenever possible. 69 * 70 * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0. If 71 * that is set, the only resize that will occur is the 0 -> SMALL_PIPE_SIZE 72 * resize which MUST occur for reverse-direction pipes when they are 73 * first used. 74 * 75 * Additional information about the current state of pipes may be obtained 76 * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail, 77 * and kern.ipc.piperesizefail. 78 * 79 * Locking rules: There are two locks present here: A mutex, used via 80 * PIPE_LOCK, and a flag, used via pipelock(). All locking is done via 81 * the flag, as mutexes can not persist over uiomove. The mutex 82 * exists only to guard access to the flag, and is not in itself a 83 * locking mechanism. Also note that there is only a single mutex for 84 * both directions of a pipe. 85 * 86 * As pipelock() may have to sleep before it can acquire the flag, it 87 * is important to reread all data after a call to pipelock(); everything 88 * in the structure may have changed. 89 */ 90 91 #include <sys/cdefs.h> 92 __FBSDID("$FreeBSD$"); 93 94 #include "opt_mac.h" 95 96 #include <sys/param.h> 97 #include <sys/systm.h> 98 #include <sys/fcntl.h> 99 #include <sys/file.h> 100 #include <sys/filedesc.h> 101 #include <sys/filio.h> 102 #include <sys/kernel.h> 103 #include <sys/lock.h> 104 #include <sys/mutex.h> 105 #include <sys/ttycom.h> 106 #include <sys/stat.h> 107 #include <sys/malloc.h> 108 #include <sys/poll.h> 109 #include <sys/selinfo.h> 110 #include <sys/signalvar.h> 111 #include <sys/sysctl.h> 112 #include <sys/sysproto.h> 113 #include <sys/pipe.h> 114 #include <sys/proc.h> 115 #include <sys/vnode.h> 116 #include <sys/uio.h> 117 #include <sys/event.h> 118 119 #include <security/mac/mac_framework.h> 120 121 #include <vm/vm.h> 122 #include <vm/vm_param.h> 123 #include <vm/vm_object.h> 124 #include <vm/vm_kern.h> 125 #include <vm/vm_extern.h> 126 #include <vm/pmap.h> 127 #include <vm/vm_map.h> 128 #include <vm/vm_page.h> 129 #include <vm/uma.h> 130 131 /* 132 * Use this define if you want to disable *fancy* VM things. Expect an 133 * approx 30% decrease in transfer rate. This could be useful for 134 * NetBSD or OpenBSD. 135 */ 136 /* #define PIPE_NODIRECT */ 137 138 /* 139 * interfaces to the outside world 140 */ 141 static fo_rdwr_t pipe_read; 142 static fo_rdwr_t pipe_write; 143 static fo_truncate_t pipe_truncate; 144 static fo_ioctl_t pipe_ioctl; 145 static fo_poll_t pipe_poll; 146 static fo_kqfilter_t pipe_kqfilter; 147 static fo_stat_t pipe_stat; 148 static fo_close_t pipe_close; 149 150 static struct fileops pipeops = { 151 .fo_read = pipe_read, 152 .fo_write = pipe_write, 153 .fo_truncate = pipe_truncate, 154 .fo_ioctl = pipe_ioctl, 155 .fo_poll = pipe_poll, 156 .fo_kqfilter = pipe_kqfilter, 157 .fo_stat = pipe_stat, 158 .fo_close = pipe_close, 159 .fo_flags = DFLAG_PASSABLE 160 }; 161 162 static void filt_pipedetach(struct knote *kn); 163 static int filt_piperead(struct knote *kn, long hint); 164 static int filt_pipewrite(struct knote *kn, long hint); 165 166 static struct filterops pipe_rfiltops = 167 { 1, NULL, filt_pipedetach, filt_piperead }; 168 static struct filterops pipe_wfiltops = 169 { 1, NULL, filt_pipedetach, filt_pipewrite }; 170 171 /* 172 * Default pipe buffer size(s), this can be kind-of large now because pipe 173 * space is pageable. The pipe code will try to maintain locality of 174 * reference for performance reasons, so small amounts of outstanding I/O 175 * will not wipe the cache. 176 */ 177 #define MINPIPESIZE (PIPE_SIZE/3) 178 #define MAXPIPESIZE (2*PIPE_SIZE/3) 179 180 static int amountpipekva; 181 static int pipefragretry; 182 static int pipeallocfail; 183 static int piperesizefail; 184 static int piperesizeallowed = 1; 185 186 SYSCTL_INT(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN, 187 &maxpipekva, 0, "Pipe KVA limit"); 188 SYSCTL_INT(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD, 189 &amountpipekva, 0, "Pipe KVA usage"); 190 SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD, 191 &pipefragretry, 0, "Pipe allocation retries due to fragmentation"); 192 SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD, 193 &pipeallocfail, 0, "Pipe allocation failures"); 194 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD, 195 &piperesizefail, 0, "Pipe resize failures"); 196 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW, 197 &piperesizeallowed, 0, "Pipe resizing allowed"); 198 199 static void pipeinit(void *dummy __unused); 200 static void pipeclose(struct pipe *cpipe); 201 static void pipe_free_kmem(struct pipe *cpipe); 202 static int pipe_create(struct pipe *pipe, int backing); 203 static __inline int pipelock(struct pipe *cpipe, int catch); 204 static __inline void pipeunlock(struct pipe *cpipe); 205 static __inline void pipeselwakeup(struct pipe *cpipe); 206 #ifndef PIPE_NODIRECT 207 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio); 208 static void pipe_destroy_write_buffer(struct pipe *wpipe); 209 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio); 210 static void pipe_clone_write_buffer(struct pipe *wpipe); 211 #endif 212 static int pipespace(struct pipe *cpipe, int size); 213 static int pipespace_new(struct pipe *cpipe, int size); 214 215 static int pipe_zone_ctor(void *mem, int size, void *arg, int flags); 216 static int pipe_zone_init(void *mem, int size, int flags); 217 static void pipe_zone_fini(void *mem, int size); 218 219 static uma_zone_t pipe_zone; 220 221 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL); 222 223 static void 224 pipeinit(void *dummy __unused) 225 { 226 227 pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair), 228 pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini, 229 UMA_ALIGN_PTR, 0); 230 KASSERT(pipe_zone != NULL, ("pipe_zone not initialized")); 231 } 232 233 static int 234 pipe_zone_ctor(void *mem, int size, void *arg, int flags) 235 { 236 struct pipepair *pp; 237 struct pipe *rpipe, *wpipe; 238 239 KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size")); 240 241 pp = (struct pipepair *)mem; 242 243 /* 244 * We zero both pipe endpoints to make sure all the kmem pointers 245 * are NULL, flag fields are zero'd, etc. We timestamp both 246 * endpoints with the same time. 247 */ 248 rpipe = &pp->pp_rpipe; 249 bzero(rpipe, sizeof(*rpipe)); 250 vfs_timestamp(&rpipe->pipe_ctime); 251 rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime; 252 253 wpipe = &pp->pp_wpipe; 254 bzero(wpipe, sizeof(*wpipe)); 255 wpipe->pipe_ctime = rpipe->pipe_ctime; 256 wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime; 257 258 rpipe->pipe_peer = wpipe; 259 rpipe->pipe_pair = pp; 260 wpipe->pipe_peer = rpipe; 261 wpipe->pipe_pair = pp; 262 263 /* 264 * Mark both endpoints as present; they will later get free'd 265 * one at a time. When both are free'd, then the whole pair 266 * is released. 267 */ 268 rpipe->pipe_present = 1; 269 wpipe->pipe_present = 1; 270 271 /* 272 * Eventually, the MAC Framework may initialize the label 273 * in ctor or init, but for now we do it elswhere to avoid 274 * blocking in ctor or init. 275 */ 276 pp->pp_label = NULL; 277 278 return (0); 279 } 280 281 static int 282 pipe_zone_init(void *mem, int size, int flags) 283 { 284 struct pipepair *pp; 285 286 KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size")); 287 288 pp = (struct pipepair *)mem; 289 290 mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_RECURSE); 291 return (0); 292 } 293 294 static void 295 pipe_zone_fini(void *mem, int size) 296 { 297 struct pipepair *pp; 298 299 KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size")); 300 301 pp = (struct pipepair *)mem; 302 303 mtx_destroy(&pp->pp_mtx); 304 } 305 306 /* 307 * The pipe system call for the DTYPE_PIPE type of pipes. If we fail, let 308 * the zone pick up the pieces via pipeclose(). 309 */ 310 /* ARGSUSED */ 311 int 312 pipe(td, uap) 313 struct thread *td; 314 struct pipe_args /* { 315 int dummy; 316 } */ *uap; 317 { 318 struct filedesc *fdp = td->td_proc->p_fd; 319 struct file *rf, *wf; 320 struct pipepair *pp; 321 struct pipe *rpipe, *wpipe; 322 int fd, error; 323 324 pp = uma_zalloc(pipe_zone, M_WAITOK); 325 #ifdef MAC 326 /* 327 * The MAC label is shared between the connected endpoints. As a 328 * result mac_pipe_init() and mac_pipe_create() are called once 329 * for the pair, and not on the endpoints. 330 */ 331 mac_pipe_init(pp); 332 mac_pipe_create(td->td_ucred, pp); 333 #endif 334 rpipe = &pp->pp_rpipe; 335 wpipe = &pp->pp_wpipe; 336 337 knlist_init(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe), NULL, NULL, 338 NULL); 339 knlist_init(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe), NULL, NULL, 340 NULL); 341 342 /* Only the forward direction pipe is backed by default */ 343 if ((error = pipe_create(rpipe, 1)) != 0 || 344 (error = pipe_create(wpipe, 0)) != 0) { 345 pipeclose(rpipe); 346 pipeclose(wpipe); 347 return (error); 348 } 349 350 rpipe->pipe_state |= PIPE_DIRECTOK; 351 wpipe->pipe_state |= PIPE_DIRECTOK; 352 353 error = falloc(td, &rf, &fd); 354 if (error) { 355 pipeclose(rpipe); 356 pipeclose(wpipe); 357 return (error); 358 } 359 /* An extra reference on `rf' has been held for us by falloc(). */ 360 td->td_retval[0] = fd; 361 362 /* 363 * Warning: once we've gotten past allocation of the fd for the 364 * read-side, we can only drop the read side via fdrop() in order 365 * to avoid races against processes which manage to dup() the read 366 * side while we are blocked trying to allocate the write side. 367 */ 368 finit(rf, FREAD | FWRITE, DTYPE_PIPE, rpipe, &pipeops); 369 error = falloc(td, &wf, &fd); 370 if (error) { 371 fdclose(fdp, rf, td->td_retval[0], td); 372 fdrop(rf, td); 373 /* rpipe has been closed by fdrop(). */ 374 pipeclose(wpipe); 375 return (error); 376 } 377 /* An extra reference on `wf' has been held for us by falloc(). */ 378 finit(wf, FREAD | FWRITE, DTYPE_PIPE, wpipe, &pipeops); 379 fdrop(wf, td); 380 td->td_retval[1] = fd; 381 fdrop(rf, td); 382 383 return (0); 384 } 385 386 /* 387 * Allocate kva for pipe circular buffer, the space is pageable 388 * This routine will 'realloc' the size of a pipe safely, if it fails 389 * it will retain the old buffer. 390 * If it fails it will return ENOMEM. 391 */ 392 static int 393 pipespace_new(cpipe, size) 394 struct pipe *cpipe; 395 int size; 396 { 397 caddr_t buffer; 398 int error, cnt, firstseg; 399 static int curfail = 0; 400 static struct timeval lastfail; 401 402 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked")); 403 KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW), 404 ("pipespace: resize of direct writes not allowed")); 405 retry: 406 cnt = cpipe->pipe_buffer.cnt; 407 if (cnt > size) 408 size = cnt; 409 410 size = round_page(size); 411 buffer = (caddr_t) vm_map_min(pipe_map); 412 413 error = vm_map_find(pipe_map, NULL, 0, 414 (vm_offset_t *) &buffer, size, 1, 415 VM_PROT_ALL, VM_PROT_ALL, 0); 416 if (error != KERN_SUCCESS) { 417 if ((cpipe->pipe_buffer.buffer == NULL) && 418 (size > SMALL_PIPE_SIZE)) { 419 size = SMALL_PIPE_SIZE; 420 pipefragretry++; 421 goto retry; 422 } 423 if (cpipe->pipe_buffer.buffer == NULL) { 424 pipeallocfail++; 425 if (ppsratecheck(&lastfail, &curfail, 1)) 426 printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n"); 427 } else { 428 piperesizefail++; 429 } 430 return (ENOMEM); 431 } 432 433 /* copy data, then free old resources if we're resizing */ 434 if (cnt > 0) { 435 if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) { 436 firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out; 437 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 438 buffer, firstseg); 439 if ((cnt - firstseg) > 0) 440 bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg], 441 cpipe->pipe_buffer.in); 442 } else { 443 bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out], 444 buffer, cnt); 445 } 446 } 447 pipe_free_kmem(cpipe); 448 cpipe->pipe_buffer.buffer = buffer; 449 cpipe->pipe_buffer.size = size; 450 cpipe->pipe_buffer.in = cnt; 451 cpipe->pipe_buffer.out = 0; 452 cpipe->pipe_buffer.cnt = cnt; 453 atomic_add_int(&amountpipekva, cpipe->pipe_buffer.size); 454 return (0); 455 } 456 457 /* 458 * Wrapper for pipespace_new() that performs locking assertions. 459 */ 460 static int 461 pipespace(cpipe, size) 462 struct pipe *cpipe; 463 int size; 464 { 465 466 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 467 ("Unlocked pipe passed to pipespace")); 468 return (pipespace_new(cpipe, size)); 469 } 470 471 /* 472 * lock a pipe for I/O, blocking other access 473 */ 474 static __inline int 475 pipelock(cpipe, catch) 476 struct pipe *cpipe; 477 int catch; 478 { 479 int error; 480 481 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 482 while (cpipe->pipe_state & PIPE_LOCKFL) { 483 cpipe->pipe_state |= PIPE_LWANT; 484 error = msleep(cpipe, PIPE_MTX(cpipe), 485 catch ? (PRIBIO | PCATCH) : PRIBIO, 486 "pipelk", 0); 487 if (error != 0) 488 return (error); 489 } 490 cpipe->pipe_state |= PIPE_LOCKFL; 491 return (0); 492 } 493 494 /* 495 * unlock a pipe I/O lock 496 */ 497 static __inline void 498 pipeunlock(cpipe) 499 struct pipe *cpipe; 500 { 501 502 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 503 KASSERT(cpipe->pipe_state & PIPE_LOCKFL, 504 ("Unlocked pipe passed to pipeunlock")); 505 cpipe->pipe_state &= ~PIPE_LOCKFL; 506 if (cpipe->pipe_state & PIPE_LWANT) { 507 cpipe->pipe_state &= ~PIPE_LWANT; 508 wakeup(cpipe); 509 } 510 } 511 512 static __inline void 513 pipeselwakeup(cpipe) 514 struct pipe *cpipe; 515 { 516 517 PIPE_LOCK_ASSERT(cpipe, MA_OWNED); 518 if (cpipe->pipe_state & PIPE_SEL) { 519 selwakeuppri(&cpipe->pipe_sel, PSOCK); 520 if (!SEL_WAITING(&cpipe->pipe_sel)) 521 cpipe->pipe_state &= ~PIPE_SEL; 522 } 523 if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio) 524 pgsigio(&cpipe->pipe_sigio, SIGIO, 0); 525 KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0); 526 } 527 528 /* 529 * Initialize and allocate VM and memory for pipe. The structure 530 * will start out zero'd from the ctor, so we just manage the kmem. 531 */ 532 static int 533 pipe_create(pipe, backing) 534 struct pipe *pipe; 535 int backing; 536 { 537 int error; 538 539 if (backing) { 540 if (amountpipekva > maxpipekva / 2) 541 error = pipespace_new(pipe, SMALL_PIPE_SIZE); 542 else 543 error = pipespace_new(pipe, PIPE_SIZE); 544 } else { 545 /* If we're not backing this pipe, no need to do anything. */ 546 error = 0; 547 } 548 return (error); 549 } 550 551 /* ARGSUSED */ 552 static int 553 pipe_read(fp, uio, active_cred, flags, td) 554 struct file *fp; 555 struct uio *uio; 556 struct ucred *active_cred; 557 struct thread *td; 558 int flags; 559 { 560 struct pipe *rpipe = fp->f_data; 561 int error; 562 int nread = 0; 563 u_int size; 564 565 PIPE_LOCK(rpipe); 566 ++rpipe->pipe_busy; 567 error = pipelock(rpipe, 1); 568 if (error) 569 goto unlocked_error; 570 571 #ifdef MAC 572 error = mac_pipe_check_read(active_cred, rpipe->pipe_pair); 573 if (error) 574 goto locked_error; 575 #endif 576 if (amountpipekva > (3 * maxpipekva) / 4) { 577 if (!(rpipe->pipe_state & PIPE_DIRECTW) && 578 (rpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 579 (rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 580 (piperesizeallowed == 1)) { 581 PIPE_UNLOCK(rpipe); 582 pipespace(rpipe, SMALL_PIPE_SIZE); 583 PIPE_LOCK(rpipe); 584 } 585 } 586 587 while (uio->uio_resid) { 588 /* 589 * normal pipe buffer receive 590 */ 591 if (rpipe->pipe_buffer.cnt > 0) { 592 size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out; 593 if (size > rpipe->pipe_buffer.cnt) 594 size = rpipe->pipe_buffer.cnt; 595 if (size > (u_int) uio->uio_resid) 596 size = (u_int) uio->uio_resid; 597 598 PIPE_UNLOCK(rpipe); 599 error = uiomove( 600 &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out], 601 size, uio); 602 PIPE_LOCK(rpipe); 603 if (error) 604 break; 605 606 rpipe->pipe_buffer.out += size; 607 if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) 608 rpipe->pipe_buffer.out = 0; 609 610 rpipe->pipe_buffer.cnt -= size; 611 612 /* 613 * If there is no more to read in the pipe, reset 614 * its pointers to the beginning. This improves 615 * cache hit stats. 616 */ 617 if (rpipe->pipe_buffer.cnt == 0) { 618 rpipe->pipe_buffer.in = 0; 619 rpipe->pipe_buffer.out = 0; 620 } 621 nread += size; 622 #ifndef PIPE_NODIRECT 623 /* 624 * Direct copy, bypassing a kernel buffer. 625 */ 626 } else if ((size = rpipe->pipe_map.cnt) && 627 (rpipe->pipe_state & PIPE_DIRECTW)) { 628 if (size > (u_int) uio->uio_resid) 629 size = (u_int) uio->uio_resid; 630 631 PIPE_UNLOCK(rpipe); 632 error = uiomove_fromphys(rpipe->pipe_map.ms, 633 rpipe->pipe_map.pos, size, uio); 634 PIPE_LOCK(rpipe); 635 if (error) 636 break; 637 nread += size; 638 rpipe->pipe_map.pos += size; 639 rpipe->pipe_map.cnt -= size; 640 if (rpipe->pipe_map.cnt == 0) { 641 rpipe->pipe_state &= ~PIPE_DIRECTW; 642 wakeup(rpipe); 643 } 644 #endif 645 } else { 646 /* 647 * detect EOF condition 648 * read returns 0 on EOF, no need to set error 649 */ 650 if (rpipe->pipe_state & PIPE_EOF) 651 break; 652 653 /* 654 * If the "write-side" has been blocked, wake it up now. 655 */ 656 if (rpipe->pipe_state & PIPE_WANTW) { 657 rpipe->pipe_state &= ~PIPE_WANTW; 658 wakeup(rpipe); 659 } 660 661 /* 662 * Break if some data was read. 663 */ 664 if (nread > 0) 665 break; 666 667 /* 668 * Unlock the pipe buffer for our remaining processing. 669 * We will either break out with an error or we will 670 * sleep and relock to loop. 671 */ 672 pipeunlock(rpipe); 673 674 /* 675 * Handle non-blocking mode operation or 676 * wait for more data. 677 */ 678 if (fp->f_flag & FNONBLOCK) { 679 error = EAGAIN; 680 } else { 681 rpipe->pipe_state |= PIPE_WANTR; 682 if ((error = msleep(rpipe, PIPE_MTX(rpipe), 683 PRIBIO | PCATCH, 684 "piperd", 0)) == 0) 685 error = pipelock(rpipe, 1); 686 } 687 if (error) 688 goto unlocked_error; 689 } 690 } 691 #ifdef MAC 692 locked_error: 693 #endif 694 pipeunlock(rpipe); 695 696 /* XXX: should probably do this before getting any locks. */ 697 if (error == 0) 698 vfs_timestamp(&rpipe->pipe_atime); 699 unlocked_error: 700 --rpipe->pipe_busy; 701 702 /* 703 * PIPE_WANT processing only makes sense if pipe_busy is 0. 704 */ 705 if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) { 706 rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW); 707 wakeup(rpipe); 708 } else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) { 709 /* 710 * Handle write blocking hysteresis. 711 */ 712 if (rpipe->pipe_state & PIPE_WANTW) { 713 rpipe->pipe_state &= ~PIPE_WANTW; 714 wakeup(rpipe); 715 } 716 } 717 718 if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF) 719 pipeselwakeup(rpipe); 720 721 PIPE_UNLOCK(rpipe); 722 return (error); 723 } 724 725 #ifndef PIPE_NODIRECT 726 /* 727 * Map the sending processes' buffer into kernel space and wire it. 728 * This is similar to a physical write operation. 729 */ 730 static int 731 pipe_build_write_buffer(wpipe, uio) 732 struct pipe *wpipe; 733 struct uio *uio; 734 { 735 pmap_t pmap; 736 u_int size; 737 int i, j; 738 vm_offset_t addr, endaddr; 739 740 PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED); 741 KASSERT(wpipe->pipe_state & PIPE_DIRECTW, 742 ("Clone attempt on non-direct write pipe!")); 743 744 size = (u_int) uio->uio_iov->iov_len; 745 if (size > wpipe->pipe_buffer.size) 746 size = wpipe->pipe_buffer.size; 747 748 pmap = vmspace_pmap(curproc->p_vmspace); 749 endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size); 750 addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base); 751 for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) { 752 /* 753 * vm_fault_quick() can sleep. Consequently, 754 * vm_page_lock_queue() and vm_page_unlock_queue() 755 * should not be performed outside of this loop. 756 */ 757 race: 758 if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0) { 759 vm_page_lock_queues(); 760 for (j = 0; j < i; j++) 761 vm_page_unhold(wpipe->pipe_map.ms[j]); 762 vm_page_unlock_queues(); 763 return (EFAULT); 764 } 765 wpipe->pipe_map.ms[i] = pmap_extract_and_hold(pmap, addr, 766 VM_PROT_READ); 767 if (wpipe->pipe_map.ms[i] == NULL) 768 goto race; 769 } 770 771 /* 772 * set up the control block 773 */ 774 wpipe->pipe_map.npages = i; 775 wpipe->pipe_map.pos = 776 ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK; 777 wpipe->pipe_map.cnt = size; 778 779 /* 780 * and update the uio data 781 */ 782 783 uio->uio_iov->iov_len -= size; 784 uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size; 785 if (uio->uio_iov->iov_len == 0) 786 uio->uio_iov++; 787 uio->uio_resid -= size; 788 uio->uio_offset += size; 789 return (0); 790 } 791 792 /* 793 * unmap and unwire the process buffer 794 */ 795 static void 796 pipe_destroy_write_buffer(wpipe) 797 struct pipe *wpipe; 798 { 799 int i; 800 801 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 802 vm_page_lock_queues(); 803 for (i = 0; i < wpipe->pipe_map.npages; i++) { 804 vm_page_unhold(wpipe->pipe_map.ms[i]); 805 } 806 vm_page_unlock_queues(); 807 wpipe->pipe_map.npages = 0; 808 } 809 810 /* 811 * In the case of a signal, the writing process might go away. This 812 * code copies the data into the circular buffer so that the source 813 * pages can be freed without loss of data. 814 */ 815 static void 816 pipe_clone_write_buffer(wpipe) 817 struct pipe *wpipe; 818 { 819 struct uio uio; 820 struct iovec iov; 821 int size; 822 int pos; 823 824 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 825 size = wpipe->pipe_map.cnt; 826 pos = wpipe->pipe_map.pos; 827 828 wpipe->pipe_buffer.in = size; 829 wpipe->pipe_buffer.out = 0; 830 wpipe->pipe_buffer.cnt = size; 831 wpipe->pipe_state &= ~PIPE_DIRECTW; 832 833 PIPE_UNLOCK(wpipe); 834 iov.iov_base = wpipe->pipe_buffer.buffer; 835 iov.iov_len = size; 836 uio.uio_iov = &iov; 837 uio.uio_iovcnt = 1; 838 uio.uio_offset = 0; 839 uio.uio_resid = size; 840 uio.uio_segflg = UIO_SYSSPACE; 841 uio.uio_rw = UIO_READ; 842 uio.uio_td = curthread; 843 uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio); 844 PIPE_LOCK(wpipe); 845 pipe_destroy_write_buffer(wpipe); 846 } 847 848 /* 849 * This implements the pipe buffer write mechanism. Note that only 850 * a direct write OR a normal pipe write can be pending at any given time. 851 * If there are any characters in the pipe buffer, the direct write will 852 * be deferred until the receiving process grabs all of the bytes from 853 * the pipe buffer. Then the direct mapping write is set-up. 854 */ 855 static int 856 pipe_direct_write(wpipe, uio) 857 struct pipe *wpipe; 858 struct uio *uio; 859 { 860 int error; 861 862 retry: 863 PIPE_LOCK_ASSERT(wpipe, MA_OWNED); 864 error = pipelock(wpipe, 1); 865 if (wpipe->pipe_state & PIPE_EOF) 866 error = EPIPE; 867 if (error) { 868 pipeunlock(wpipe); 869 goto error1; 870 } 871 while (wpipe->pipe_state & PIPE_DIRECTW) { 872 if (wpipe->pipe_state & PIPE_WANTR) { 873 wpipe->pipe_state &= ~PIPE_WANTR; 874 wakeup(wpipe); 875 } 876 pipeselwakeup(wpipe); 877 wpipe->pipe_state |= PIPE_WANTW; 878 pipeunlock(wpipe); 879 error = msleep(wpipe, PIPE_MTX(wpipe), 880 PRIBIO | PCATCH, "pipdww", 0); 881 if (error) 882 goto error1; 883 else 884 goto retry; 885 } 886 wpipe->pipe_map.cnt = 0; /* transfer not ready yet */ 887 if (wpipe->pipe_buffer.cnt > 0) { 888 if (wpipe->pipe_state & PIPE_WANTR) { 889 wpipe->pipe_state &= ~PIPE_WANTR; 890 wakeup(wpipe); 891 } 892 pipeselwakeup(wpipe); 893 wpipe->pipe_state |= PIPE_WANTW; 894 pipeunlock(wpipe); 895 error = msleep(wpipe, PIPE_MTX(wpipe), 896 PRIBIO | PCATCH, "pipdwc", 0); 897 if (error) 898 goto error1; 899 else 900 goto retry; 901 } 902 903 wpipe->pipe_state |= PIPE_DIRECTW; 904 905 PIPE_UNLOCK(wpipe); 906 error = pipe_build_write_buffer(wpipe, uio); 907 PIPE_LOCK(wpipe); 908 if (error) { 909 wpipe->pipe_state &= ~PIPE_DIRECTW; 910 pipeunlock(wpipe); 911 goto error1; 912 } 913 914 error = 0; 915 while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) { 916 if (wpipe->pipe_state & PIPE_EOF) { 917 pipe_destroy_write_buffer(wpipe); 918 pipeselwakeup(wpipe); 919 pipeunlock(wpipe); 920 error = EPIPE; 921 goto error1; 922 } 923 if (wpipe->pipe_state & PIPE_WANTR) { 924 wpipe->pipe_state &= ~PIPE_WANTR; 925 wakeup(wpipe); 926 } 927 pipeselwakeup(wpipe); 928 pipeunlock(wpipe); 929 error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH, 930 "pipdwt", 0); 931 pipelock(wpipe, 0); 932 } 933 934 if (wpipe->pipe_state & PIPE_EOF) 935 error = EPIPE; 936 if (wpipe->pipe_state & PIPE_DIRECTW) { 937 /* 938 * this bit of trickery substitutes a kernel buffer for 939 * the process that might be going away. 940 */ 941 pipe_clone_write_buffer(wpipe); 942 } else { 943 pipe_destroy_write_buffer(wpipe); 944 } 945 pipeunlock(wpipe); 946 return (error); 947 948 error1: 949 wakeup(wpipe); 950 return (error); 951 } 952 #endif 953 954 static int 955 pipe_write(fp, uio, active_cred, flags, td) 956 struct file *fp; 957 struct uio *uio; 958 struct ucred *active_cred; 959 struct thread *td; 960 int flags; 961 { 962 int error = 0; 963 int desiredsize, orig_resid; 964 struct pipe *wpipe, *rpipe; 965 966 rpipe = fp->f_data; 967 wpipe = rpipe->pipe_peer; 968 969 PIPE_LOCK(rpipe); 970 error = pipelock(wpipe, 1); 971 if (error) { 972 PIPE_UNLOCK(rpipe); 973 return (error); 974 } 975 /* 976 * detect loss of pipe read side, issue SIGPIPE if lost. 977 */ 978 if ((!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 979 pipeunlock(wpipe); 980 PIPE_UNLOCK(rpipe); 981 return (EPIPE); 982 } 983 #ifdef MAC 984 error = mac_pipe_check_write(active_cred, wpipe->pipe_pair); 985 if (error) { 986 pipeunlock(wpipe); 987 PIPE_UNLOCK(rpipe); 988 return (error); 989 } 990 #endif 991 ++wpipe->pipe_busy; 992 993 /* Choose a larger size if it's advantageous */ 994 desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size); 995 while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) { 996 if (piperesizeallowed != 1) 997 break; 998 if (amountpipekva > maxpipekva / 2) 999 break; 1000 if (desiredsize == BIG_PIPE_SIZE) 1001 break; 1002 desiredsize = desiredsize * 2; 1003 } 1004 1005 /* Choose a smaller size if we're in a OOM situation */ 1006 if ((amountpipekva > (3 * maxpipekva) / 4) && 1007 (wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) && 1008 (wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) && 1009 (piperesizeallowed == 1)) 1010 desiredsize = SMALL_PIPE_SIZE; 1011 1012 /* Resize if the above determined that a new size was necessary */ 1013 if ((desiredsize != wpipe->pipe_buffer.size) && 1014 ((wpipe->pipe_state & PIPE_DIRECTW) == 0)) { 1015 PIPE_UNLOCK(wpipe); 1016 pipespace(wpipe, desiredsize); 1017 PIPE_LOCK(wpipe); 1018 } 1019 if (wpipe->pipe_buffer.size == 0) { 1020 /* 1021 * This can only happen for reverse direction use of pipes 1022 * in a complete OOM situation. 1023 */ 1024 error = ENOMEM; 1025 --wpipe->pipe_busy; 1026 pipeunlock(wpipe); 1027 PIPE_UNLOCK(wpipe); 1028 return (error); 1029 } 1030 1031 pipeunlock(wpipe); 1032 1033 orig_resid = uio->uio_resid; 1034 1035 while (uio->uio_resid) { 1036 int space; 1037 1038 pipelock(wpipe, 0); 1039 if (wpipe->pipe_state & PIPE_EOF) { 1040 pipeunlock(wpipe); 1041 error = EPIPE; 1042 break; 1043 } 1044 #ifndef PIPE_NODIRECT 1045 /* 1046 * If the transfer is large, we can gain performance if 1047 * we do process-to-process copies directly. 1048 * If the write is non-blocking, we don't use the 1049 * direct write mechanism. 1050 * 1051 * The direct write mechanism will detect the reader going 1052 * away on us. 1053 */ 1054 if (uio->uio_segflg == UIO_USERSPACE && 1055 uio->uio_iov->iov_len >= PIPE_MINDIRECT && 1056 wpipe->pipe_buffer.size >= PIPE_MINDIRECT && 1057 (fp->f_flag & FNONBLOCK) == 0) { 1058 pipeunlock(wpipe); 1059 error = pipe_direct_write(wpipe, uio); 1060 if (error) 1061 break; 1062 continue; 1063 } 1064 #endif 1065 1066 /* 1067 * Pipe buffered writes cannot be coincidental with 1068 * direct writes. We wait until the currently executing 1069 * direct write is completed before we start filling the 1070 * pipe buffer. We break out if a signal occurs or the 1071 * reader goes away. 1072 */ 1073 if (wpipe->pipe_state & PIPE_DIRECTW) { 1074 if (wpipe->pipe_state & PIPE_WANTR) { 1075 wpipe->pipe_state &= ~PIPE_WANTR; 1076 wakeup(wpipe); 1077 } 1078 pipeselwakeup(wpipe); 1079 wpipe->pipe_state |= PIPE_WANTW; 1080 pipeunlock(wpipe); 1081 error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, 1082 "pipbww", 0); 1083 if (error) 1084 break; 1085 else 1086 continue; 1087 } 1088 1089 space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1090 1091 /* Writes of size <= PIPE_BUF must be atomic. */ 1092 if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF)) 1093 space = 0; 1094 1095 if (space > 0) { 1096 int size; /* Transfer size */ 1097 int segsize; /* first segment to transfer */ 1098 1099 /* 1100 * Transfer size is minimum of uio transfer 1101 * and free space in pipe buffer. 1102 */ 1103 if (space > uio->uio_resid) 1104 size = uio->uio_resid; 1105 else 1106 size = space; 1107 /* 1108 * First segment to transfer is minimum of 1109 * transfer size and contiguous space in 1110 * pipe buffer. If first segment to transfer 1111 * is less than the transfer size, we've got 1112 * a wraparound in the buffer. 1113 */ 1114 segsize = wpipe->pipe_buffer.size - 1115 wpipe->pipe_buffer.in; 1116 if (segsize > size) 1117 segsize = size; 1118 1119 /* Transfer first segment */ 1120 1121 PIPE_UNLOCK(rpipe); 1122 error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in], 1123 segsize, uio); 1124 PIPE_LOCK(rpipe); 1125 1126 if (error == 0 && segsize < size) { 1127 KASSERT(wpipe->pipe_buffer.in + segsize == 1128 wpipe->pipe_buffer.size, 1129 ("Pipe buffer wraparound disappeared")); 1130 /* 1131 * Transfer remaining part now, to 1132 * support atomic writes. Wraparound 1133 * happened. 1134 */ 1135 1136 PIPE_UNLOCK(rpipe); 1137 error = uiomove( 1138 &wpipe->pipe_buffer.buffer[0], 1139 size - segsize, uio); 1140 PIPE_LOCK(rpipe); 1141 } 1142 if (error == 0) { 1143 wpipe->pipe_buffer.in += size; 1144 if (wpipe->pipe_buffer.in >= 1145 wpipe->pipe_buffer.size) { 1146 KASSERT(wpipe->pipe_buffer.in == 1147 size - segsize + 1148 wpipe->pipe_buffer.size, 1149 ("Expected wraparound bad")); 1150 wpipe->pipe_buffer.in = size - segsize; 1151 } 1152 1153 wpipe->pipe_buffer.cnt += size; 1154 KASSERT(wpipe->pipe_buffer.cnt <= 1155 wpipe->pipe_buffer.size, 1156 ("Pipe buffer overflow")); 1157 } 1158 pipeunlock(wpipe); 1159 if (error != 0) 1160 break; 1161 } else { 1162 /* 1163 * If the "read-side" has been blocked, wake it up now. 1164 */ 1165 if (wpipe->pipe_state & PIPE_WANTR) { 1166 wpipe->pipe_state &= ~PIPE_WANTR; 1167 wakeup(wpipe); 1168 } 1169 1170 /* 1171 * don't block on non-blocking I/O 1172 */ 1173 if (fp->f_flag & FNONBLOCK) { 1174 error = EAGAIN; 1175 pipeunlock(wpipe); 1176 break; 1177 } 1178 1179 /* 1180 * We have no more space and have something to offer, 1181 * wake up select/poll. 1182 */ 1183 pipeselwakeup(wpipe); 1184 1185 wpipe->pipe_state |= PIPE_WANTW; 1186 pipeunlock(wpipe); 1187 error = msleep(wpipe, PIPE_MTX(rpipe), 1188 PRIBIO | PCATCH, "pipewr", 0); 1189 if (error != 0) 1190 break; 1191 } 1192 } 1193 1194 pipelock(wpipe, 0); 1195 --wpipe->pipe_busy; 1196 1197 if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) { 1198 wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR); 1199 wakeup(wpipe); 1200 } else if (wpipe->pipe_buffer.cnt > 0) { 1201 /* 1202 * If we have put any characters in the buffer, we wake up 1203 * the reader. 1204 */ 1205 if (wpipe->pipe_state & PIPE_WANTR) { 1206 wpipe->pipe_state &= ~PIPE_WANTR; 1207 wakeup(wpipe); 1208 } 1209 } 1210 1211 /* 1212 * Don't return EPIPE if I/O was successful 1213 */ 1214 if ((wpipe->pipe_buffer.cnt == 0) && 1215 (uio->uio_resid == 0) && 1216 (error == EPIPE)) { 1217 error = 0; 1218 } 1219 1220 if (error == 0) 1221 vfs_timestamp(&wpipe->pipe_mtime); 1222 1223 /* 1224 * We have something to offer, 1225 * wake up select/poll. 1226 */ 1227 if (wpipe->pipe_buffer.cnt) 1228 pipeselwakeup(wpipe); 1229 1230 pipeunlock(wpipe); 1231 PIPE_UNLOCK(rpipe); 1232 return (error); 1233 } 1234 1235 /* ARGSUSED */ 1236 static int 1237 pipe_truncate(fp, length, active_cred, td) 1238 struct file *fp; 1239 off_t length; 1240 struct ucred *active_cred; 1241 struct thread *td; 1242 { 1243 1244 return (EINVAL); 1245 } 1246 1247 /* 1248 * we implement a very minimal set of ioctls for compatibility with sockets. 1249 */ 1250 static int 1251 pipe_ioctl(fp, cmd, data, active_cred, td) 1252 struct file *fp; 1253 u_long cmd; 1254 void *data; 1255 struct ucred *active_cred; 1256 struct thread *td; 1257 { 1258 struct pipe *mpipe = fp->f_data; 1259 int error; 1260 1261 PIPE_LOCK(mpipe); 1262 1263 #ifdef MAC 1264 error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data); 1265 if (error) { 1266 PIPE_UNLOCK(mpipe); 1267 return (error); 1268 } 1269 #endif 1270 1271 error = 0; 1272 switch (cmd) { 1273 1274 case FIONBIO: 1275 break; 1276 1277 case FIOASYNC: 1278 if (*(int *)data) { 1279 mpipe->pipe_state |= PIPE_ASYNC; 1280 } else { 1281 mpipe->pipe_state &= ~PIPE_ASYNC; 1282 } 1283 break; 1284 1285 case FIONREAD: 1286 if (mpipe->pipe_state & PIPE_DIRECTW) 1287 *(int *)data = mpipe->pipe_map.cnt; 1288 else 1289 *(int *)data = mpipe->pipe_buffer.cnt; 1290 break; 1291 1292 case FIOSETOWN: 1293 PIPE_UNLOCK(mpipe); 1294 error = fsetown(*(int *)data, &mpipe->pipe_sigio); 1295 goto out_unlocked; 1296 1297 case FIOGETOWN: 1298 *(int *)data = fgetown(&mpipe->pipe_sigio); 1299 break; 1300 1301 /* This is deprecated, FIOSETOWN should be used instead. */ 1302 case TIOCSPGRP: 1303 PIPE_UNLOCK(mpipe); 1304 error = fsetown(-(*(int *)data), &mpipe->pipe_sigio); 1305 goto out_unlocked; 1306 1307 /* This is deprecated, FIOGETOWN should be used instead. */ 1308 case TIOCGPGRP: 1309 *(int *)data = -fgetown(&mpipe->pipe_sigio); 1310 break; 1311 1312 default: 1313 error = ENOTTY; 1314 break; 1315 } 1316 PIPE_UNLOCK(mpipe); 1317 out_unlocked: 1318 return (error); 1319 } 1320 1321 static int 1322 pipe_poll(fp, events, active_cred, td) 1323 struct file *fp; 1324 int events; 1325 struct ucred *active_cred; 1326 struct thread *td; 1327 { 1328 struct pipe *rpipe = fp->f_data; 1329 struct pipe *wpipe; 1330 int revents = 0; 1331 #ifdef MAC 1332 int error; 1333 #endif 1334 1335 wpipe = rpipe->pipe_peer; 1336 PIPE_LOCK(rpipe); 1337 #ifdef MAC 1338 error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair); 1339 if (error) 1340 goto locked_error; 1341 #endif 1342 if (events & (POLLIN | POLLRDNORM)) 1343 if ((rpipe->pipe_state & PIPE_DIRECTW) || 1344 (rpipe->pipe_buffer.cnt > 0) || 1345 (rpipe->pipe_state & PIPE_EOF)) 1346 revents |= events & (POLLIN | POLLRDNORM); 1347 1348 if (events & (POLLOUT | POLLWRNORM)) 1349 if (!wpipe->pipe_present || (wpipe->pipe_state & PIPE_EOF) || 1350 (((wpipe->pipe_state & PIPE_DIRECTW) == 0) && 1351 (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF)) 1352 revents |= events & (POLLOUT | POLLWRNORM); 1353 1354 if ((rpipe->pipe_state & PIPE_EOF) || 1355 (!wpipe->pipe_present) || 1356 (wpipe->pipe_state & PIPE_EOF)) 1357 revents |= POLLHUP; 1358 1359 if (revents == 0) { 1360 if (events & (POLLIN | POLLRDNORM)) { 1361 selrecord(td, &rpipe->pipe_sel); 1362 if (SEL_WAITING(&rpipe->pipe_sel)) 1363 rpipe->pipe_state |= PIPE_SEL; 1364 } 1365 1366 if (events & (POLLOUT | POLLWRNORM)) { 1367 selrecord(td, &wpipe->pipe_sel); 1368 if (SEL_WAITING(&wpipe->pipe_sel)) 1369 wpipe->pipe_state |= PIPE_SEL; 1370 } 1371 } 1372 #ifdef MAC 1373 locked_error: 1374 #endif 1375 PIPE_UNLOCK(rpipe); 1376 1377 return (revents); 1378 } 1379 1380 /* 1381 * We shouldn't need locks here as we're doing a read and this should 1382 * be a natural race. 1383 */ 1384 static int 1385 pipe_stat(fp, ub, active_cred, td) 1386 struct file *fp; 1387 struct stat *ub; 1388 struct ucred *active_cred; 1389 struct thread *td; 1390 { 1391 struct pipe *pipe = fp->f_data; 1392 #ifdef MAC 1393 int error; 1394 1395 PIPE_LOCK(pipe); 1396 error = mac_pipe_check_stat(active_cred, pipe->pipe_pair); 1397 PIPE_UNLOCK(pipe); 1398 if (error) 1399 return (error); 1400 #endif 1401 bzero(ub, sizeof(*ub)); 1402 ub->st_mode = S_IFIFO; 1403 ub->st_blksize = PAGE_SIZE; 1404 if (pipe->pipe_state & PIPE_DIRECTW) 1405 ub->st_size = pipe->pipe_map.cnt; 1406 else 1407 ub->st_size = pipe->pipe_buffer.cnt; 1408 ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize; 1409 ub->st_atimespec = pipe->pipe_atime; 1410 ub->st_mtimespec = pipe->pipe_mtime; 1411 ub->st_ctimespec = pipe->pipe_ctime; 1412 ub->st_uid = fp->f_cred->cr_uid; 1413 ub->st_gid = fp->f_cred->cr_gid; 1414 /* 1415 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen. 1416 * XXX (st_dev, st_ino) should be unique. 1417 */ 1418 return (0); 1419 } 1420 1421 /* ARGSUSED */ 1422 static int 1423 pipe_close(fp, td) 1424 struct file *fp; 1425 struct thread *td; 1426 { 1427 struct pipe *cpipe = fp->f_data; 1428 1429 fp->f_ops = &badfileops; 1430 fp->f_data = NULL; 1431 funsetown(&cpipe->pipe_sigio); 1432 pipeclose(cpipe); 1433 return (0); 1434 } 1435 1436 static void 1437 pipe_free_kmem(cpipe) 1438 struct pipe *cpipe; 1439 { 1440 1441 KASSERT(!mtx_owned(PIPE_MTX(cpipe)), 1442 ("pipe_free_kmem: pipe mutex locked")); 1443 1444 if (cpipe->pipe_buffer.buffer != NULL) { 1445 atomic_subtract_int(&amountpipekva, cpipe->pipe_buffer.size); 1446 vm_map_remove(pipe_map, 1447 (vm_offset_t)cpipe->pipe_buffer.buffer, 1448 (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size); 1449 cpipe->pipe_buffer.buffer = NULL; 1450 } 1451 #ifndef PIPE_NODIRECT 1452 { 1453 cpipe->pipe_map.cnt = 0; 1454 cpipe->pipe_map.pos = 0; 1455 cpipe->pipe_map.npages = 0; 1456 } 1457 #endif 1458 } 1459 1460 /* 1461 * shutdown the pipe 1462 */ 1463 static void 1464 pipeclose(cpipe) 1465 struct pipe *cpipe; 1466 { 1467 struct pipepair *pp; 1468 struct pipe *ppipe; 1469 1470 KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL")); 1471 1472 PIPE_LOCK(cpipe); 1473 pipelock(cpipe, 0); 1474 pp = cpipe->pipe_pair; 1475 1476 pipeselwakeup(cpipe); 1477 1478 /* 1479 * If the other side is blocked, wake it up saying that 1480 * we want to close it down. 1481 */ 1482 cpipe->pipe_state |= PIPE_EOF; 1483 while (cpipe->pipe_busy) { 1484 wakeup(cpipe); 1485 cpipe->pipe_state |= PIPE_WANT; 1486 pipeunlock(cpipe); 1487 msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0); 1488 pipelock(cpipe, 0); 1489 } 1490 1491 1492 /* 1493 * Disconnect from peer, if any. 1494 */ 1495 ppipe = cpipe->pipe_peer; 1496 if (ppipe->pipe_present != 0) { 1497 pipeselwakeup(ppipe); 1498 1499 ppipe->pipe_state |= PIPE_EOF; 1500 wakeup(ppipe); 1501 KNOTE_LOCKED(&ppipe->pipe_sel.si_note, 0); 1502 } 1503 1504 /* 1505 * Mark this endpoint as free. Release kmem resources. We 1506 * don't mark this endpoint as unused until we've finished 1507 * doing that, or the pipe might disappear out from under 1508 * us. 1509 */ 1510 PIPE_UNLOCK(cpipe); 1511 pipe_free_kmem(cpipe); 1512 PIPE_LOCK(cpipe); 1513 cpipe->pipe_present = 0; 1514 pipeunlock(cpipe); 1515 knlist_clear(&cpipe->pipe_sel.si_note, 1); 1516 knlist_destroy(&cpipe->pipe_sel.si_note); 1517 1518 /* 1519 * If both endpoints are now closed, release the memory for the 1520 * pipe pair. If not, unlock. 1521 */ 1522 if (ppipe->pipe_present == 0) { 1523 PIPE_UNLOCK(cpipe); 1524 #ifdef MAC 1525 mac_pipe_destroy(pp); 1526 #endif 1527 uma_zfree(pipe_zone, cpipe->pipe_pair); 1528 } else 1529 PIPE_UNLOCK(cpipe); 1530 } 1531 1532 /*ARGSUSED*/ 1533 static int 1534 pipe_kqfilter(struct file *fp, struct knote *kn) 1535 { 1536 struct pipe *cpipe; 1537 1538 cpipe = kn->kn_fp->f_data; 1539 PIPE_LOCK(cpipe); 1540 switch (kn->kn_filter) { 1541 case EVFILT_READ: 1542 kn->kn_fop = &pipe_rfiltops; 1543 break; 1544 case EVFILT_WRITE: 1545 kn->kn_fop = &pipe_wfiltops; 1546 if (!cpipe->pipe_peer->pipe_present) { 1547 /* other end of pipe has been closed */ 1548 PIPE_UNLOCK(cpipe); 1549 return (EPIPE); 1550 } 1551 cpipe = cpipe->pipe_peer; 1552 break; 1553 default: 1554 PIPE_UNLOCK(cpipe); 1555 return (EINVAL); 1556 } 1557 1558 knlist_add(&cpipe->pipe_sel.si_note, kn, 1); 1559 PIPE_UNLOCK(cpipe); 1560 return (0); 1561 } 1562 1563 static void 1564 filt_pipedetach(struct knote *kn) 1565 { 1566 struct pipe *cpipe = (struct pipe *)kn->kn_fp->f_data; 1567 1568 PIPE_LOCK(cpipe); 1569 if (kn->kn_filter == EVFILT_WRITE) { 1570 if (!cpipe->pipe_peer->pipe_present) { 1571 PIPE_UNLOCK(cpipe); 1572 return; 1573 } 1574 cpipe = cpipe->pipe_peer; 1575 } 1576 knlist_remove(&cpipe->pipe_sel.si_note, kn, 1); 1577 PIPE_UNLOCK(cpipe); 1578 } 1579 1580 /*ARGSUSED*/ 1581 static int 1582 filt_piperead(struct knote *kn, long hint) 1583 { 1584 struct pipe *rpipe = kn->kn_fp->f_data; 1585 struct pipe *wpipe = rpipe->pipe_peer; 1586 int ret; 1587 1588 PIPE_LOCK(rpipe); 1589 kn->kn_data = rpipe->pipe_buffer.cnt; 1590 if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW)) 1591 kn->kn_data = rpipe->pipe_map.cnt; 1592 1593 if ((rpipe->pipe_state & PIPE_EOF) || 1594 (!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 1595 kn->kn_flags |= EV_EOF; 1596 PIPE_UNLOCK(rpipe); 1597 return (1); 1598 } 1599 ret = kn->kn_data > 0; 1600 PIPE_UNLOCK(rpipe); 1601 return ret; 1602 } 1603 1604 /*ARGSUSED*/ 1605 static int 1606 filt_pipewrite(struct knote *kn, long hint) 1607 { 1608 struct pipe *rpipe = kn->kn_fp->f_data; 1609 struct pipe *wpipe = rpipe->pipe_peer; 1610 1611 PIPE_LOCK(rpipe); 1612 if ((!wpipe->pipe_present) || (wpipe->pipe_state & PIPE_EOF)) { 1613 kn->kn_data = 0; 1614 kn->kn_flags |= EV_EOF; 1615 PIPE_UNLOCK(rpipe); 1616 return (1); 1617 } 1618 kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt; 1619 if (wpipe->pipe_state & PIPE_DIRECTW) 1620 kn->kn_data = 0; 1621 1622 PIPE_UNLOCK(rpipe); 1623 return (kn->kn_data >= PIPE_BUF); 1624 } 1625