xref: /freebsd/sys/kern/sys_pipe.c (revision 4ce386ff25d77954b8cfa11534f632172e848244)
1 /*-
2  * Copyright (c) 1996 John S. Dyson
3  * Copyright (c) 2012 Giovanni Trematerra
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice immediately at the beginning of the file, without modification,
11  *    this list of conditions, and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. Absolutely no warranty of function or purpose is made by the author
16  *    John S. Dyson.
17  * 4. Modifications may be freely made to this file if the above conditions
18  *    are met.
19  */
20 
21 /*
22  * This file contains a high-performance replacement for the socket-based
23  * pipes scheme originally used in FreeBSD/4.4Lite.  It does not support
24  * all features of sockets, but does do everything that pipes normally
25  * do.
26  */
27 
28 /*
29  * This code has two modes of operation, a small write mode and a large
30  * write mode.  The small write mode acts like conventional pipes with
31  * a kernel buffer.  If the buffer is less than PIPE_MINDIRECT, then the
32  * "normal" pipe buffering is done.  If the buffer is between PIPE_MINDIRECT
33  * and PIPE_SIZE in size, the sending process pins the underlying pages in
34  * memory, and the receiving process copies directly from these pinned pages
35  * in the sending process.
36  *
37  * If the sending process receives a signal, it is possible that it will
38  * go away, and certainly its address space can change, because control
39  * is returned back to the user-mode side.  In that case, the pipe code
40  * arranges to copy the buffer supplied by the user process, to a pageable
41  * kernel buffer, and the receiving process will grab the data from the
42  * pageable kernel buffer.  Since signals don't happen all that often,
43  * the copy operation is normally eliminated.
44  *
45  * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
46  * happen for small transfers so that the system will not spend all of
47  * its time context switching.
48  *
49  * In order to limit the resource use of pipes, two sysctls exist:
50  *
51  * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable
52  * address space available to us in pipe_map. This value is normally
53  * autotuned, but may also be loader tuned.
54  *
55  * kern.ipc.pipekva - This read-only sysctl tracks the current amount of
56  * memory in use by pipes.
57  *
58  * Based on how large pipekva is relative to maxpipekva, the following
59  * will happen:
60  *
61  * 0% - 50%:
62  *     New pipes are given 16K of memory backing, pipes may dynamically
63  *     grow to as large as 64K where needed.
64  * 50% - 75%:
65  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
66  *     existing pipes may NOT grow.
67  * 75% - 100%:
68  *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
69  *     existing pipes will be shrunk down to 4K whenever possible.
70  *
71  * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0.  If
72  * that is set,  the only resize that will occur is the 0 -> SMALL_PIPE_SIZE
73  * resize which MUST occur for reverse-direction pipes when they are
74  * first used.
75  *
76  * Additional information about the current state of pipes may be obtained
77  * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail,
78  * and kern.ipc.piperesizefail.
79  *
80  * Locking rules:  There are two locks present here:  A mutex, used via
81  * PIPE_LOCK, and a flag, used via pipelock().  All locking is done via
82  * the flag, as mutexes can not persist over uiomove.  The mutex
83  * exists only to guard access to the flag, and is not in itself a
84  * locking mechanism.  Also note that there is only a single mutex for
85  * both directions of a pipe.
86  *
87  * As pipelock() may have to sleep before it can acquire the flag, it
88  * is important to reread all data after a call to pipelock(); everything
89  * in the structure may have changed.
90  */
91 
92 #include <sys/cdefs.h>
93 __FBSDID("$FreeBSD$");
94 
95 #include <sys/param.h>
96 #include <sys/systm.h>
97 #include <sys/conf.h>
98 #include <sys/fcntl.h>
99 #include <sys/file.h>
100 #include <sys/filedesc.h>
101 #include <sys/filio.h>
102 #include <sys/kernel.h>
103 #include <sys/lock.h>
104 #include <sys/mutex.h>
105 #include <sys/ttycom.h>
106 #include <sys/stat.h>
107 #include <sys/malloc.h>
108 #include <sys/poll.h>
109 #include <sys/selinfo.h>
110 #include <sys/signalvar.h>
111 #include <sys/syscallsubr.h>
112 #include <sys/sysctl.h>
113 #include <sys/sysproto.h>
114 #include <sys/pipe.h>
115 #include <sys/proc.h>
116 #include <sys/vnode.h>
117 #include <sys/uio.h>
118 #include <sys/user.h>
119 #include <sys/event.h>
120 
121 #include <security/mac/mac_framework.h>
122 
123 #include <vm/vm.h>
124 #include <vm/vm_param.h>
125 #include <vm/vm_object.h>
126 #include <vm/vm_kern.h>
127 #include <vm/vm_extern.h>
128 #include <vm/pmap.h>
129 #include <vm/vm_map.h>
130 #include <vm/vm_page.h>
131 #include <vm/uma.h>
132 
133 /*
134  * Use this define if you want to disable *fancy* VM things.  Expect an
135  * approx 30% decrease in transfer rate.  This could be useful for
136  * NetBSD or OpenBSD.
137  */
138 /* #define PIPE_NODIRECT */
139 
140 #define PIPE_PEER(pipe)	\
141 	(((pipe)->pipe_state & PIPE_NAMED) ? (pipe) : ((pipe)->pipe_peer))
142 
143 /*
144  * interfaces to the outside world
145  */
146 static fo_rdwr_t	pipe_read;
147 static fo_rdwr_t	pipe_write;
148 static fo_truncate_t	pipe_truncate;
149 static fo_ioctl_t	pipe_ioctl;
150 static fo_poll_t	pipe_poll;
151 static fo_kqfilter_t	pipe_kqfilter;
152 static fo_stat_t	pipe_stat;
153 static fo_close_t	pipe_close;
154 static fo_chmod_t	pipe_chmod;
155 static fo_chown_t	pipe_chown;
156 static fo_fill_kinfo_t	pipe_fill_kinfo;
157 
158 struct fileops pipeops = {
159 	.fo_read = pipe_read,
160 	.fo_write = pipe_write,
161 	.fo_truncate = pipe_truncate,
162 	.fo_ioctl = pipe_ioctl,
163 	.fo_poll = pipe_poll,
164 	.fo_kqfilter = pipe_kqfilter,
165 	.fo_stat = pipe_stat,
166 	.fo_close = pipe_close,
167 	.fo_chmod = pipe_chmod,
168 	.fo_chown = pipe_chown,
169 	.fo_sendfile = invfo_sendfile,
170 	.fo_fill_kinfo = pipe_fill_kinfo,
171 	.fo_flags = DFLAG_PASSABLE
172 };
173 
174 static void	filt_pipedetach(struct knote *kn);
175 static void	filt_pipedetach_notsup(struct knote *kn);
176 static int	filt_pipenotsup(struct knote *kn, long hint);
177 static int	filt_piperead(struct knote *kn, long hint);
178 static int	filt_pipewrite(struct knote *kn, long hint);
179 
180 static struct filterops pipe_nfiltops = {
181 	.f_isfd = 1,
182 	.f_detach = filt_pipedetach_notsup,
183 	.f_event = filt_pipenotsup
184 };
185 static struct filterops pipe_rfiltops = {
186 	.f_isfd = 1,
187 	.f_detach = filt_pipedetach,
188 	.f_event = filt_piperead
189 };
190 static struct filterops pipe_wfiltops = {
191 	.f_isfd = 1,
192 	.f_detach = filt_pipedetach,
193 	.f_event = filt_pipewrite
194 };
195 
196 /*
197  * Default pipe buffer size(s), this can be kind-of large now because pipe
198  * space is pageable.  The pipe code will try to maintain locality of
199  * reference for performance reasons, so small amounts of outstanding I/O
200  * will not wipe the cache.
201  */
202 #define MINPIPESIZE (PIPE_SIZE/3)
203 #define MAXPIPESIZE (2*PIPE_SIZE/3)
204 
205 static long amountpipekva;
206 static int pipefragretry;
207 static int pipeallocfail;
208 static int piperesizefail;
209 static int piperesizeallowed = 1;
210 
211 SYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
212 	   &maxpipekva, 0, "Pipe KVA limit");
213 SYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD,
214 	   &amountpipekva, 0, "Pipe KVA usage");
215 SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD,
216 	  &pipefragretry, 0, "Pipe allocation retries due to fragmentation");
217 SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD,
218 	  &pipeallocfail, 0, "Pipe allocation failures");
219 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD,
220 	  &piperesizefail, 0, "Pipe resize failures");
221 SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW,
222 	  &piperesizeallowed, 0, "Pipe resizing allowed");
223 
224 static void pipeinit(void *dummy __unused);
225 static void pipeclose(struct pipe *cpipe);
226 static void pipe_free_kmem(struct pipe *cpipe);
227 static void pipe_create(struct pipe *pipe, int backing);
228 static void pipe_paircreate(struct thread *td, struct pipepair **p_pp);
229 static __inline int pipelock(struct pipe *cpipe, int catch);
230 static __inline void pipeunlock(struct pipe *cpipe);
231 #ifndef PIPE_NODIRECT
232 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
233 static void pipe_destroy_write_buffer(struct pipe *wpipe);
234 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
235 static void pipe_clone_write_buffer(struct pipe *wpipe);
236 #endif
237 static int pipespace(struct pipe *cpipe, int size);
238 static int pipespace_new(struct pipe *cpipe, int size);
239 
240 static int	pipe_zone_ctor(void *mem, int size, void *arg, int flags);
241 static int	pipe_zone_init(void *mem, int size, int flags);
242 static void	pipe_zone_fini(void *mem, int size);
243 
244 static uma_zone_t pipe_zone;
245 static struct unrhdr *pipeino_unr;
246 static dev_t pipedev_ino;
247 
248 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
249 
250 static void
251 pipeinit(void *dummy __unused)
252 {
253 
254 	pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair),
255 	    pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini,
256 	    UMA_ALIGN_PTR, 0);
257 	KASSERT(pipe_zone != NULL, ("pipe_zone not initialized"));
258 	pipeino_unr = new_unrhdr(1, INT32_MAX, NULL);
259 	KASSERT(pipeino_unr != NULL, ("pipe fake inodes not initialized"));
260 	pipedev_ino = devfs_alloc_cdp_inode();
261 	KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized"));
262 }
263 
264 static int
265 pipe_zone_ctor(void *mem, int size, void *arg, int flags)
266 {
267 	struct pipepair *pp;
268 	struct pipe *rpipe, *wpipe;
269 
270 	KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size"));
271 
272 	pp = (struct pipepair *)mem;
273 
274 	/*
275 	 * We zero both pipe endpoints to make sure all the kmem pointers
276 	 * are NULL, flag fields are zero'd, etc.  We timestamp both
277 	 * endpoints with the same time.
278 	 */
279 	rpipe = &pp->pp_rpipe;
280 	bzero(rpipe, sizeof(*rpipe));
281 	vfs_timestamp(&rpipe->pipe_ctime);
282 	rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime;
283 
284 	wpipe = &pp->pp_wpipe;
285 	bzero(wpipe, sizeof(*wpipe));
286 	wpipe->pipe_ctime = rpipe->pipe_ctime;
287 	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;
288 
289 	rpipe->pipe_peer = wpipe;
290 	rpipe->pipe_pair = pp;
291 	wpipe->pipe_peer = rpipe;
292 	wpipe->pipe_pair = pp;
293 
294 	/*
295 	 * Mark both endpoints as present; they will later get free'd
296 	 * one at a time.  When both are free'd, then the whole pair
297 	 * is released.
298 	 */
299 	rpipe->pipe_present = PIPE_ACTIVE;
300 	wpipe->pipe_present = PIPE_ACTIVE;
301 
302 	/*
303 	 * Eventually, the MAC Framework may initialize the label
304 	 * in ctor or init, but for now we do it elswhere to avoid
305 	 * blocking in ctor or init.
306 	 */
307 	pp->pp_label = NULL;
308 
309 	return (0);
310 }
311 
312 static int
313 pipe_zone_init(void *mem, int size, int flags)
314 {
315 	struct pipepair *pp;
316 
317 	KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size"));
318 
319 	pp = (struct pipepair *)mem;
320 
321 	mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_NEW);
322 	return (0);
323 }
324 
325 static void
326 pipe_zone_fini(void *mem, int size)
327 {
328 	struct pipepair *pp;
329 
330 	KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size"));
331 
332 	pp = (struct pipepair *)mem;
333 
334 	mtx_destroy(&pp->pp_mtx);
335 }
336 
337 static void
338 pipe_paircreate(struct thread *td, struct pipepair **p_pp)
339 {
340 	struct pipepair *pp;
341 	struct pipe *rpipe, *wpipe;
342 
343 	*p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK);
344 #ifdef MAC
345 	/*
346 	 * The MAC label is shared between the connected endpoints.  As a
347 	 * result mac_pipe_init() and mac_pipe_create() are called once
348 	 * for the pair, and not on the endpoints.
349 	 */
350 	mac_pipe_init(pp);
351 	mac_pipe_create(td->td_ucred, pp);
352 #endif
353 	rpipe = &pp->pp_rpipe;
354 	wpipe = &pp->pp_wpipe;
355 
356 	knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe));
357 	knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));
358 
359 	/* Only the forward direction pipe is backed by default */
360 	pipe_create(rpipe, 1);
361 	pipe_create(wpipe, 0);
362 
363 	rpipe->pipe_state |= PIPE_DIRECTOK;
364 	wpipe->pipe_state |= PIPE_DIRECTOK;
365 }
366 
367 void
368 pipe_named_ctor(struct pipe **ppipe, struct thread *td)
369 {
370 	struct pipepair *pp;
371 
372 	pipe_paircreate(td, &pp);
373 	pp->pp_rpipe.pipe_state |= PIPE_NAMED;
374 	*ppipe = &pp->pp_rpipe;
375 }
376 
377 void
378 pipe_dtor(struct pipe *dpipe)
379 {
380 	struct pipe *peer;
381 	ino_t ino;
382 
383 	ino = dpipe->pipe_ino;
384 	peer = (dpipe->pipe_state & PIPE_NAMED) != 0 ? dpipe->pipe_peer : NULL;
385 	funsetown(&dpipe->pipe_sigio);
386 	pipeclose(dpipe);
387 	if (peer != NULL) {
388 		funsetown(&peer->pipe_sigio);
389 		pipeclose(peer);
390 	}
391 	if (ino != 0 && ino != (ino_t)-1)
392 		free_unr(pipeino_unr, ino);
393 }
394 
395 /*
396  * The pipe system call for the DTYPE_PIPE type of pipes.  If we fail, let
397  * the zone pick up the pieces via pipeclose().
398  */
399 int
400 kern_pipe(struct thread *td, int fildes[2])
401 {
402 
403 	return (kern_pipe2(td, fildes, 0));
404 }
405 
406 int
407 kern_pipe2(struct thread *td, int fildes[2], int flags)
408 {
409 	struct filedesc *fdp;
410 	struct file *rf, *wf;
411 	struct pipe *rpipe, *wpipe;
412 	struct pipepair *pp;
413 	int fd, fflags, error;
414 
415 	fdp = td->td_proc->p_fd;
416 	pipe_paircreate(td, &pp);
417 	rpipe = &pp->pp_rpipe;
418 	wpipe = &pp->pp_wpipe;
419 	error = falloc(td, &rf, &fd, flags);
420 	if (error) {
421 		pipeclose(rpipe);
422 		pipeclose(wpipe);
423 		return (error);
424 	}
425 	/* An extra reference on `rf' has been held for us by falloc(). */
426 	fildes[0] = fd;
427 
428 	fflags = FREAD | FWRITE;
429 	if ((flags & O_NONBLOCK) != 0)
430 		fflags |= FNONBLOCK;
431 
432 	/*
433 	 * Warning: once we've gotten past allocation of the fd for the
434 	 * read-side, we can only drop the read side via fdrop() in order
435 	 * to avoid races against processes which manage to dup() the read
436 	 * side while we are blocked trying to allocate the write side.
437 	 */
438 	finit(rf, fflags, DTYPE_PIPE, rpipe, &pipeops);
439 	error = falloc(td, &wf, &fd, flags);
440 	if (error) {
441 		fdclose(fdp, rf, fildes[0], td);
442 		fdrop(rf, td);
443 		/* rpipe has been closed by fdrop(). */
444 		pipeclose(wpipe);
445 		return (error);
446 	}
447 	/* An extra reference on `wf' has been held for us by falloc(). */
448 	finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);
449 	fdrop(wf, td);
450 	fildes[1] = fd;
451 	fdrop(rf, td);
452 
453 	return (0);
454 }
455 
456 /* ARGSUSED */
457 int
458 sys_pipe(struct thread *td, struct pipe_args *uap)
459 {
460 	int error;
461 	int fildes[2];
462 
463 	error = kern_pipe(td, fildes);
464 	if (error)
465 		return (error);
466 
467 	td->td_retval[0] = fildes[0];
468 	td->td_retval[1] = fildes[1];
469 
470 	return (0);
471 }
472 
473 int
474 sys_pipe2(struct thread *td, struct pipe2_args *uap)
475 {
476 	int error, fildes[2];
477 
478 	if (uap->flags & ~(O_CLOEXEC | O_NONBLOCK))
479 		return (EINVAL);
480 	error = kern_pipe2(td, fildes, uap->flags);
481 	if (error)
482 		return (error);
483 	error = copyout(fildes, uap->fildes, 2 * sizeof(int));
484 	if (error) {
485 		(void)kern_close(td, fildes[0]);
486 		(void)kern_close(td, fildes[1]);
487 	}
488 	return (error);
489 }
490 
491 /*
492  * Allocate kva for pipe circular buffer, the space is pageable
493  * This routine will 'realloc' the size of a pipe safely, if it fails
494  * it will retain the old buffer.
495  * If it fails it will return ENOMEM.
496  */
497 static int
498 pipespace_new(cpipe, size)
499 	struct pipe *cpipe;
500 	int size;
501 {
502 	caddr_t buffer;
503 	int error, cnt, firstseg;
504 	static int curfail = 0;
505 	static struct timeval lastfail;
506 
507 	KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked"));
508 	KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW),
509 		("pipespace: resize of direct writes not allowed"));
510 retry:
511 	cnt = cpipe->pipe_buffer.cnt;
512 	if (cnt > size)
513 		size = cnt;
514 
515 	size = round_page(size);
516 	buffer = (caddr_t) vm_map_min(pipe_map);
517 
518 	error = vm_map_find(pipe_map, NULL, 0,
519 		(vm_offset_t *) &buffer, size, 0, VMFS_ANY_SPACE,
520 		VM_PROT_ALL, VM_PROT_ALL, 0);
521 	if (error != KERN_SUCCESS) {
522 		if ((cpipe->pipe_buffer.buffer == NULL) &&
523 			(size > SMALL_PIPE_SIZE)) {
524 			size = SMALL_PIPE_SIZE;
525 			pipefragretry++;
526 			goto retry;
527 		}
528 		if (cpipe->pipe_buffer.buffer == NULL) {
529 			pipeallocfail++;
530 			if (ppsratecheck(&lastfail, &curfail, 1))
531 				printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n");
532 		} else {
533 			piperesizefail++;
534 		}
535 		return (ENOMEM);
536 	}
537 
538 	/* copy data, then free old resources if we're resizing */
539 	if (cnt > 0) {
540 		if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) {
541 			firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out;
542 			bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
543 				buffer, firstseg);
544 			if ((cnt - firstseg) > 0)
545 				bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg],
546 					cpipe->pipe_buffer.in);
547 		} else {
548 			bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
549 				buffer, cnt);
550 		}
551 	}
552 	pipe_free_kmem(cpipe);
553 	cpipe->pipe_buffer.buffer = buffer;
554 	cpipe->pipe_buffer.size = size;
555 	cpipe->pipe_buffer.in = cnt;
556 	cpipe->pipe_buffer.out = 0;
557 	cpipe->pipe_buffer.cnt = cnt;
558 	atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size);
559 	return (0);
560 }
561 
562 /*
563  * Wrapper for pipespace_new() that performs locking assertions.
564  */
565 static int
566 pipespace(cpipe, size)
567 	struct pipe *cpipe;
568 	int size;
569 {
570 
571 	KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
572 		("Unlocked pipe passed to pipespace"));
573 	return (pipespace_new(cpipe, size));
574 }
575 
576 /*
577  * lock a pipe for I/O, blocking other access
578  */
579 static __inline int
580 pipelock(cpipe, catch)
581 	struct pipe *cpipe;
582 	int catch;
583 {
584 	int error;
585 
586 	PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
587 	while (cpipe->pipe_state & PIPE_LOCKFL) {
588 		cpipe->pipe_state |= PIPE_LWANT;
589 		error = msleep(cpipe, PIPE_MTX(cpipe),
590 		    catch ? (PRIBIO | PCATCH) : PRIBIO,
591 		    "pipelk", 0);
592 		if (error != 0)
593 			return (error);
594 	}
595 	cpipe->pipe_state |= PIPE_LOCKFL;
596 	return (0);
597 }
598 
599 /*
600  * unlock a pipe I/O lock
601  */
602 static __inline void
603 pipeunlock(cpipe)
604 	struct pipe *cpipe;
605 {
606 
607 	PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
608 	KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
609 		("Unlocked pipe passed to pipeunlock"));
610 	cpipe->pipe_state &= ~PIPE_LOCKFL;
611 	if (cpipe->pipe_state & PIPE_LWANT) {
612 		cpipe->pipe_state &= ~PIPE_LWANT;
613 		wakeup(cpipe);
614 	}
615 }
616 
617 void
618 pipeselwakeup(cpipe)
619 	struct pipe *cpipe;
620 {
621 
622 	PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
623 	if (cpipe->pipe_state & PIPE_SEL) {
624 		selwakeuppri(&cpipe->pipe_sel, PSOCK);
625 		if (!SEL_WAITING(&cpipe->pipe_sel))
626 			cpipe->pipe_state &= ~PIPE_SEL;
627 	}
628 	if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
629 		pgsigio(&cpipe->pipe_sigio, SIGIO, 0);
630 	KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0);
631 }
632 
633 /*
634  * Initialize and allocate VM and memory for pipe.  The structure
635  * will start out zero'd from the ctor, so we just manage the kmem.
636  */
637 static void
638 pipe_create(pipe, backing)
639 	struct pipe *pipe;
640 	int backing;
641 {
642 
643 	if (backing) {
644 		/*
645 		 * Note that these functions can fail if pipe map is exhausted
646 		 * (as a result of too many pipes created), but we ignore the
647 		 * error as it is not fatal and could be provoked by
648 		 * unprivileged users. The only consequence is worse performance
649 		 * with given pipe.
650 		 */
651 		if (amountpipekva > maxpipekva / 2)
652 			(void)pipespace_new(pipe, SMALL_PIPE_SIZE);
653 		else
654 			(void)pipespace_new(pipe, PIPE_SIZE);
655 	}
656 
657 	pipe->pipe_ino = -1;
658 }
659 
660 /* ARGSUSED */
661 static int
662 pipe_read(fp, uio, active_cred, flags, td)
663 	struct file *fp;
664 	struct uio *uio;
665 	struct ucred *active_cred;
666 	struct thread *td;
667 	int flags;
668 {
669 	struct pipe *rpipe;
670 	int error;
671 	int nread = 0;
672 	int size;
673 
674 	rpipe = fp->f_data;
675 	PIPE_LOCK(rpipe);
676 	++rpipe->pipe_busy;
677 	error = pipelock(rpipe, 1);
678 	if (error)
679 		goto unlocked_error;
680 
681 #ifdef MAC
682 	error = mac_pipe_check_read(active_cred, rpipe->pipe_pair);
683 	if (error)
684 		goto locked_error;
685 #endif
686 	if (amountpipekva > (3 * maxpipekva) / 4) {
687 		if (!(rpipe->pipe_state & PIPE_DIRECTW) &&
688 			(rpipe->pipe_buffer.size > SMALL_PIPE_SIZE) &&
689 			(rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) &&
690 			(piperesizeallowed == 1)) {
691 			PIPE_UNLOCK(rpipe);
692 			pipespace(rpipe, SMALL_PIPE_SIZE);
693 			PIPE_LOCK(rpipe);
694 		}
695 	}
696 
697 	while (uio->uio_resid) {
698 		/*
699 		 * normal pipe buffer receive
700 		 */
701 		if (rpipe->pipe_buffer.cnt > 0) {
702 			size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
703 			if (size > rpipe->pipe_buffer.cnt)
704 				size = rpipe->pipe_buffer.cnt;
705 			if (size > uio->uio_resid)
706 				size = uio->uio_resid;
707 
708 			PIPE_UNLOCK(rpipe);
709 			error = uiomove(
710 			    &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],
711 			    size, uio);
712 			PIPE_LOCK(rpipe);
713 			if (error)
714 				break;
715 
716 			rpipe->pipe_buffer.out += size;
717 			if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
718 				rpipe->pipe_buffer.out = 0;
719 
720 			rpipe->pipe_buffer.cnt -= size;
721 
722 			/*
723 			 * If there is no more to read in the pipe, reset
724 			 * its pointers to the beginning.  This improves
725 			 * cache hit stats.
726 			 */
727 			if (rpipe->pipe_buffer.cnt == 0) {
728 				rpipe->pipe_buffer.in = 0;
729 				rpipe->pipe_buffer.out = 0;
730 			}
731 			nread += size;
732 #ifndef PIPE_NODIRECT
733 		/*
734 		 * Direct copy, bypassing a kernel buffer.
735 		 */
736 		} else if ((size = rpipe->pipe_map.cnt) &&
737 			   (rpipe->pipe_state & PIPE_DIRECTW)) {
738 			if (size > uio->uio_resid)
739 				size = (u_int) uio->uio_resid;
740 
741 			PIPE_UNLOCK(rpipe);
742 			error = uiomove_fromphys(rpipe->pipe_map.ms,
743 			    rpipe->pipe_map.pos, size, uio);
744 			PIPE_LOCK(rpipe);
745 			if (error)
746 				break;
747 			nread += size;
748 			rpipe->pipe_map.pos += size;
749 			rpipe->pipe_map.cnt -= size;
750 			if (rpipe->pipe_map.cnt == 0) {
751 				rpipe->pipe_state &= ~(PIPE_DIRECTW|PIPE_WANTW);
752 				wakeup(rpipe);
753 			}
754 #endif
755 		} else {
756 			/*
757 			 * detect EOF condition
758 			 * read returns 0 on EOF, no need to set error
759 			 */
760 			if (rpipe->pipe_state & PIPE_EOF)
761 				break;
762 
763 			/*
764 			 * If the "write-side" has been blocked, wake it up now.
765 			 */
766 			if (rpipe->pipe_state & PIPE_WANTW) {
767 				rpipe->pipe_state &= ~PIPE_WANTW;
768 				wakeup(rpipe);
769 			}
770 
771 			/*
772 			 * Break if some data was read.
773 			 */
774 			if (nread > 0)
775 				break;
776 
777 			/*
778 			 * Unlock the pipe buffer for our remaining processing.
779 			 * We will either break out with an error or we will
780 			 * sleep and relock to loop.
781 			 */
782 			pipeunlock(rpipe);
783 
784 			/*
785 			 * Handle non-blocking mode operation or
786 			 * wait for more data.
787 			 */
788 			if (fp->f_flag & FNONBLOCK) {
789 				error = EAGAIN;
790 			} else {
791 				rpipe->pipe_state |= PIPE_WANTR;
792 				if ((error = msleep(rpipe, PIPE_MTX(rpipe),
793 				    PRIBIO | PCATCH,
794 				    "piperd", 0)) == 0)
795 					error = pipelock(rpipe, 1);
796 			}
797 			if (error)
798 				goto unlocked_error;
799 		}
800 	}
801 #ifdef MAC
802 locked_error:
803 #endif
804 	pipeunlock(rpipe);
805 
806 	/* XXX: should probably do this before getting any locks. */
807 	if (error == 0)
808 		vfs_timestamp(&rpipe->pipe_atime);
809 unlocked_error:
810 	--rpipe->pipe_busy;
811 
812 	/*
813 	 * PIPE_WANT processing only makes sense if pipe_busy is 0.
814 	 */
815 	if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
816 		rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
817 		wakeup(rpipe);
818 	} else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
819 		/*
820 		 * Handle write blocking hysteresis.
821 		 */
822 		if (rpipe->pipe_state & PIPE_WANTW) {
823 			rpipe->pipe_state &= ~PIPE_WANTW;
824 			wakeup(rpipe);
825 		}
826 	}
827 
828 	if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF)
829 		pipeselwakeup(rpipe);
830 
831 	PIPE_UNLOCK(rpipe);
832 	return (error);
833 }
834 
835 #ifndef PIPE_NODIRECT
836 /*
837  * Map the sending processes' buffer into kernel space and wire it.
838  * This is similar to a physical write operation.
839  */
840 static int
841 pipe_build_write_buffer(wpipe, uio)
842 	struct pipe *wpipe;
843 	struct uio *uio;
844 {
845 	u_int size;
846 	int i;
847 
848 	PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED);
849 	KASSERT(wpipe->pipe_state & PIPE_DIRECTW,
850 		("Clone attempt on non-direct write pipe!"));
851 
852 	if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)
853                 size = wpipe->pipe_buffer.size;
854 	else
855                 size = uio->uio_iov->iov_len;
856 
857 	if ((i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
858 	    (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ,
859 	    wpipe->pipe_map.ms, PIPENPAGES)) < 0)
860 		return (EFAULT);
861 
862 /*
863  * set up the control block
864  */
865 	wpipe->pipe_map.npages = i;
866 	wpipe->pipe_map.pos =
867 	    ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK;
868 	wpipe->pipe_map.cnt = size;
869 
870 /*
871  * and update the uio data
872  */
873 
874 	uio->uio_iov->iov_len -= size;
875 	uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size;
876 	if (uio->uio_iov->iov_len == 0)
877 		uio->uio_iov++;
878 	uio->uio_resid -= size;
879 	uio->uio_offset += size;
880 	return (0);
881 }
882 
883 /*
884  * unmap and unwire the process buffer
885  */
886 static void
887 pipe_destroy_write_buffer(wpipe)
888 	struct pipe *wpipe;
889 {
890 
891 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
892 	vm_page_unhold_pages(wpipe->pipe_map.ms, wpipe->pipe_map.npages);
893 	wpipe->pipe_map.npages = 0;
894 }
895 
896 /*
897  * In the case of a signal, the writing process might go away.  This
898  * code copies the data into the circular buffer so that the source
899  * pages can be freed without loss of data.
900  */
901 static void
902 pipe_clone_write_buffer(wpipe)
903 	struct pipe *wpipe;
904 {
905 	struct uio uio;
906 	struct iovec iov;
907 	int size;
908 	int pos;
909 
910 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
911 	size = wpipe->pipe_map.cnt;
912 	pos = wpipe->pipe_map.pos;
913 
914 	wpipe->pipe_buffer.in = size;
915 	wpipe->pipe_buffer.out = 0;
916 	wpipe->pipe_buffer.cnt = size;
917 	wpipe->pipe_state &= ~PIPE_DIRECTW;
918 
919 	PIPE_UNLOCK(wpipe);
920 	iov.iov_base = wpipe->pipe_buffer.buffer;
921 	iov.iov_len = size;
922 	uio.uio_iov = &iov;
923 	uio.uio_iovcnt = 1;
924 	uio.uio_offset = 0;
925 	uio.uio_resid = size;
926 	uio.uio_segflg = UIO_SYSSPACE;
927 	uio.uio_rw = UIO_READ;
928 	uio.uio_td = curthread;
929 	uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio);
930 	PIPE_LOCK(wpipe);
931 	pipe_destroy_write_buffer(wpipe);
932 }
933 
934 /*
935  * This implements the pipe buffer write mechanism.  Note that only
936  * a direct write OR a normal pipe write can be pending at any given time.
937  * If there are any characters in the pipe buffer, the direct write will
938  * be deferred until the receiving process grabs all of the bytes from
939  * the pipe buffer.  Then the direct mapping write is set-up.
940  */
941 static int
942 pipe_direct_write(wpipe, uio)
943 	struct pipe *wpipe;
944 	struct uio *uio;
945 {
946 	int error;
947 
948 retry:
949 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
950 	error = pipelock(wpipe, 1);
951 	if (wpipe->pipe_state & PIPE_EOF)
952 		error = EPIPE;
953 	if (error) {
954 		pipeunlock(wpipe);
955 		goto error1;
956 	}
957 	while (wpipe->pipe_state & PIPE_DIRECTW) {
958 		if (wpipe->pipe_state & PIPE_WANTR) {
959 			wpipe->pipe_state &= ~PIPE_WANTR;
960 			wakeup(wpipe);
961 		}
962 		pipeselwakeup(wpipe);
963 		wpipe->pipe_state |= PIPE_WANTW;
964 		pipeunlock(wpipe);
965 		error = msleep(wpipe, PIPE_MTX(wpipe),
966 		    PRIBIO | PCATCH, "pipdww", 0);
967 		if (error)
968 			goto error1;
969 		else
970 			goto retry;
971 	}
972 	wpipe->pipe_map.cnt = 0;	/* transfer not ready yet */
973 	if (wpipe->pipe_buffer.cnt > 0) {
974 		if (wpipe->pipe_state & PIPE_WANTR) {
975 			wpipe->pipe_state &= ~PIPE_WANTR;
976 			wakeup(wpipe);
977 		}
978 		pipeselwakeup(wpipe);
979 		wpipe->pipe_state |= PIPE_WANTW;
980 		pipeunlock(wpipe);
981 		error = msleep(wpipe, PIPE_MTX(wpipe),
982 		    PRIBIO | PCATCH, "pipdwc", 0);
983 		if (error)
984 			goto error1;
985 		else
986 			goto retry;
987 	}
988 
989 	wpipe->pipe_state |= PIPE_DIRECTW;
990 
991 	PIPE_UNLOCK(wpipe);
992 	error = pipe_build_write_buffer(wpipe, uio);
993 	PIPE_LOCK(wpipe);
994 	if (error) {
995 		wpipe->pipe_state &= ~PIPE_DIRECTW;
996 		pipeunlock(wpipe);
997 		goto error1;
998 	}
999 
1000 	error = 0;
1001 	while (!error && (wpipe->pipe_state & PIPE_DIRECTW)) {
1002 		if (wpipe->pipe_state & PIPE_EOF) {
1003 			pipe_destroy_write_buffer(wpipe);
1004 			pipeselwakeup(wpipe);
1005 			pipeunlock(wpipe);
1006 			error = EPIPE;
1007 			goto error1;
1008 		}
1009 		if (wpipe->pipe_state & PIPE_WANTR) {
1010 			wpipe->pipe_state &= ~PIPE_WANTR;
1011 			wakeup(wpipe);
1012 		}
1013 		pipeselwakeup(wpipe);
1014 		wpipe->pipe_state |= PIPE_WANTW;
1015 		pipeunlock(wpipe);
1016 		error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,
1017 		    "pipdwt", 0);
1018 		pipelock(wpipe, 0);
1019 	}
1020 
1021 	if (wpipe->pipe_state & PIPE_EOF)
1022 		error = EPIPE;
1023 	if (wpipe->pipe_state & PIPE_DIRECTW) {
1024 		/*
1025 		 * this bit of trickery substitutes a kernel buffer for
1026 		 * the process that might be going away.
1027 		 */
1028 		pipe_clone_write_buffer(wpipe);
1029 	} else {
1030 		pipe_destroy_write_buffer(wpipe);
1031 	}
1032 	pipeunlock(wpipe);
1033 	return (error);
1034 
1035 error1:
1036 	wakeup(wpipe);
1037 	return (error);
1038 }
1039 #endif
1040 
1041 static int
1042 pipe_write(fp, uio, active_cred, flags, td)
1043 	struct file *fp;
1044 	struct uio *uio;
1045 	struct ucred *active_cred;
1046 	struct thread *td;
1047 	int flags;
1048 {
1049 	int error = 0;
1050 	int desiredsize;
1051 	ssize_t orig_resid;
1052 	struct pipe *wpipe, *rpipe;
1053 
1054 	rpipe = fp->f_data;
1055 	wpipe = PIPE_PEER(rpipe);
1056 	PIPE_LOCK(rpipe);
1057 	error = pipelock(wpipe, 1);
1058 	if (error) {
1059 		PIPE_UNLOCK(rpipe);
1060 		return (error);
1061 	}
1062 	/*
1063 	 * detect loss of pipe read side, issue SIGPIPE if lost.
1064 	 */
1065 	if (wpipe->pipe_present != PIPE_ACTIVE ||
1066 	    (wpipe->pipe_state & PIPE_EOF)) {
1067 		pipeunlock(wpipe);
1068 		PIPE_UNLOCK(rpipe);
1069 		return (EPIPE);
1070 	}
1071 #ifdef MAC
1072 	error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);
1073 	if (error) {
1074 		pipeunlock(wpipe);
1075 		PIPE_UNLOCK(rpipe);
1076 		return (error);
1077 	}
1078 #endif
1079 	++wpipe->pipe_busy;
1080 
1081 	/* Choose a larger size if it's advantageous */
1082 	desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);
1083 	while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {
1084 		if (piperesizeallowed != 1)
1085 			break;
1086 		if (amountpipekva > maxpipekva / 2)
1087 			break;
1088 		if (desiredsize == BIG_PIPE_SIZE)
1089 			break;
1090 		desiredsize = desiredsize * 2;
1091 	}
1092 
1093 	/* Choose a smaller size if we're in a OOM situation */
1094 	if ((amountpipekva > (3 * maxpipekva) / 4) &&
1095 		(wpipe->pipe_buffer.size > SMALL_PIPE_SIZE) &&
1096 		(wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE) &&
1097 		(piperesizeallowed == 1))
1098 		desiredsize = SMALL_PIPE_SIZE;
1099 
1100 	/* Resize if the above determined that a new size was necessary */
1101 	if ((desiredsize != wpipe->pipe_buffer.size) &&
1102 		((wpipe->pipe_state & PIPE_DIRECTW) == 0)) {
1103 		PIPE_UNLOCK(wpipe);
1104 		pipespace(wpipe, desiredsize);
1105 		PIPE_LOCK(wpipe);
1106 	}
1107 	if (wpipe->pipe_buffer.size == 0) {
1108 		/*
1109 		 * This can only happen for reverse direction use of pipes
1110 		 * in a complete OOM situation.
1111 		 */
1112 		error = ENOMEM;
1113 		--wpipe->pipe_busy;
1114 		pipeunlock(wpipe);
1115 		PIPE_UNLOCK(wpipe);
1116 		return (error);
1117 	}
1118 
1119 	pipeunlock(wpipe);
1120 
1121 	orig_resid = uio->uio_resid;
1122 
1123 	while (uio->uio_resid) {
1124 		int space;
1125 
1126 		pipelock(wpipe, 0);
1127 		if (wpipe->pipe_state & PIPE_EOF) {
1128 			pipeunlock(wpipe);
1129 			error = EPIPE;
1130 			break;
1131 		}
1132 #ifndef PIPE_NODIRECT
1133 		/*
1134 		 * If the transfer is large, we can gain performance if
1135 		 * we do process-to-process copies directly.
1136 		 * If the write is non-blocking, we don't use the
1137 		 * direct write mechanism.
1138 		 *
1139 		 * The direct write mechanism will detect the reader going
1140 		 * away on us.
1141 		 */
1142 		if (uio->uio_segflg == UIO_USERSPACE &&
1143 		    uio->uio_iov->iov_len >= PIPE_MINDIRECT &&
1144 		    wpipe->pipe_buffer.size >= PIPE_MINDIRECT &&
1145 		    (fp->f_flag & FNONBLOCK) == 0) {
1146 			pipeunlock(wpipe);
1147 			error = pipe_direct_write(wpipe, uio);
1148 			if (error)
1149 				break;
1150 			continue;
1151 		}
1152 #endif
1153 
1154 		/*
1155 		 * Pipe buffered writes cannot be coincidental with
1156 		 * direct writes.  We wait until the currently executing
1157 		 * direct write is completed before we start filling the
1158 		 * pipe buffer.  We break out if a signal occurs or the
1159 		 * reader goes away.
1160 		 */
1161 		if (wpipe->pipe_state & PIPE_DIRECTW) {
1162 			if (wpipe->pipe_state & PIPE_WANTR) {
1163 				wpipe->pipe_state &= ~PIPE_WANTR;
1164 				wakeup(wpipe);
1165 			}
1166 			pipeselwakeup(wpipe);
1167 			wpipe->pipe_state |= PIPE_WANTW;
1168 			pipeunlock(wpipe);
1169 			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,
1170 			    "pipbww", 0);
1171 			if (error)
1172 				break;
1173 			else
1174 				continue;
1175 		}
1176 
1177 		space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1178 
1179 		/* Writes of size <= PIPE_BUF must be atomic. */
1180 		if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF))
1181 			space = 0;
1182 
1183 		if (space > 0) {
1184 			int size;	/* Transfer size */
1185 			int segsize;	/* first segment to transfer */
1186 
1187 			/*
1188 			 * Transfer size is minimum of uio transfer
1189 			 * and free space in pipe buffer.
1190 			 */
1191 			if (space > uio->uio_resid)
1192 				size = uio->uio_resid;
1193 			else
1194 				size = space;
1195 			/*
1196 			 * First segment to transfer is minimum of
1197 			 * transfer size and contiguous space in
1198 			 * pipe buffer.  If first segment to transfer
1199 			 * is less than the transfer size, we've got
1200 			 * a wraparound in the buffer.
1201 			 */
1202 			segsize = wpipe->pipe_buffer.size -
1203 				wpipe->pipe_buffer.in;
1204 			if (segsize > size)
1205 				segsize = size;
1206 
1207 			/* Transfer first segment */
1208 
1209 			PIPE_UNLOCK(rpipe);
1210 			error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],
1211 					segsize, uio);
1212 			PIPE_LOCK(rpipe);
1213 
1214 			if (error == 0 && segsize < size) {
1215 				KASSERT(wpipe->pipe_buffer.in + segsize ==
1216 					wpipe->pipe_buffer.size,
1217 					("Pipe buffer wraparound disappeared"));
1218 				/*
1219 				 * Transfer remaining part now, to
1220 				 * support atomic writes.  Wraparound
1221 				 * happened.
1222 				 */
1223 
1224 				PIPE_UNLOCK(rpipe);
1225 				error = uiomove(
1226 				    &wpipe->pipe_buffer.buffer[0],
1227 				    size - segsize, uio);
1228 				PIPE_LOCK(rpipe);
1229 			}
1230 			if (error == 0) {
1231 				wpipe->pipe_buffer.in += size;
1232 				if (wpipe->pipe_buffer.in >=
1233 				    wpipe->pipe_buffer.size) {
1234 					KASSERT(wpipe->pipe_buffer.in ==
1235 						size - segsize +
1236 						wpipe->pipe_buffer.size,
1237 						("Expected wraparound bad"));
1238 					wpipe->pipe_buffer.in = size - segsize;
1239 				}
1240 
1241 				wpipe->pipe_buffer.cnt += size;
1242 				KASSERT(wpipe->pipe_buffer.cnt <=
1243 					wpipe->pipe_buffer.size,
1244 					("Pipe buffer overflow"));
1245 			}
1246 			pipeunlock(wpipe);
1247 			if (error != 0)
1248 				break;
1249 		} else {
1250 			/*
1251 			 * If the "read-side" has been blocked, wake it up now.
1252 			 */
1253 			if (wpipe->pipe_state & PIPE_WANTR) {
1254 				wpipe->pipe_state &= ~PIPE_WANTR;
1255 				wakeup(wpipe);
1256 			}
1257 
1258 			/*
1259 			 * don't block on non-blocking I/O
1260 			 */
1261 			if (fp->f_flag & FNONBLOCK) {
1262 				error = EAGAIN;
1263 				pipeunlock(wpipe);
1264 				break;
1265 			}
1266 
1267 			/*
1268 			 * We have no more space and have something to offer,
1269 			 * wake up select/poll.
1270 			 */
1271 			pipeselwakeup(wpipe);
1272 
1273 			wpipe->pipe_state |= PIPE_WANTW;
1274 			pipeunlock(wpipe);
1275 			error = msleep(wpipe, PIPE_MTX(rpipe),
1276 			    PRIBIO | PCATCH, "pipewr", 0);
1277 			if (error != 0)
1278 				break;
1279 		}
1280 	}
1281 
1282 	pipelock(wpipe, 0);
1283 	--wpipe->pipe_busy;
1284 
1285 	if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
1286 		wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
1287 		wakeup(wpipe);
1288 	} else if (wpipe->pipe_buffer.cnt > 0) {
1289 		/*
1290 		 * If we have put any characters in the buffer, we wake up
1291 		 * the reader.
1292 		 */
1293 		if (wpipe->pipe_state & PIPE_WANTR) {
1294 			wpipe->pipe_state &= ~PIPE_WANTR;
1295 			wakeup(wpipe);
1296 		}
1297 	}
1298 
1299 	/*
1300 	 * Don't return EPIPE if any byte was written.
1301 	 * EINTR and other interrupts are handled by generic I/O layer.
1302 	 * Do not pretend that I/O succeeded for obvious user error
1303 	 * like EFAULT.
1304 	 */
1305 	if (uio->uio_resid != orig_resid && error == EPIPE)
1306 		error = 0;
1307 
1308 	if (error == 0)
1309 		vfs_timestamp(&wpipe->pipe_mtime);
1310 
1311 	/*
1312 	 * We have something to offer,
1313 	 * wake up select/poll.
1314 	 */
1315 	if (wpipe->pipe_buffer.cnt)
1316 		pipeselwakeup(wpipe);
1317 
1318 	pipeunlock(wpipe);
1319 	PIPE_UNLOCK(rpipe);
1320 	return (error);
1321 }
1322 
1323 /* ARGSUSED */
1324 static int
1325 pipe_truncate(fp, length, active_cred, td)
1326 	struct file *fp;
1327 	off_t length;
1328 	struct ucred *active_cred;
1329 	struct thread *td;
1330 {
1331 	struct pipe *cpipe;
1332 	int error;
1333 
1334 	cpipe = fp->f_data;
1335 	if (cpipe->pipe_state & PIPE_NAMED)
1336 		error = vnops.fo_truncate(fp, length, active_cred, td);
1337 	else
1338 		error = invfo_truncate(fp, length, active_cred, td);
1339 	return (error);
1340 }
1341 
1342 /*
1343  * we implement a very minimal set of ioctls for compatibility with sockets.
1344  */
1345 static int
1346 pipe_ioctl(fp, cmd, data, active_cred, td)
1347 	struct file *fp;
1348 	u_long cmd;
1349 	void *data;
1350 	struct ucred *active_cred;
1351 	struct thread *td;
1352 {
1353 	struct pipe *mpipe = fp->f_data;
1354 	int error;
1355 
1356 	PIPE_LOCK(mpipe);
1357 
1358 #ifdef MAC
1359 	error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data);
1360 	if (error) {
1361 		PIPE_UNLOCK(mpipe);
1362 		return (error);
1363 	}
1364 #endif
1365 
1366 	error = 0;
1367 	switch (cmd) {
1368 
1369 	case FIONBIO:
1370 		break;
1371 
1372 	case FIOASYNC:
1373 		if (*(int *)data) {
1374 			mpipe->pipe_state |= PIPE_ASYNC;
1375 		} else {
1376 			mpipe->pipe_state &= ~PIPE_ASYNC;
1377 		}
1378 		break;
1379 
1380 	case FIONREAD:
1381 		if (!(fp->f_flag & FREAD)) {
1382 			*(int *)data = 0;
1383 			PIPE_UNLOCK(mpipe);
1384 			return (0);
1385 		}
1386 		if (mpipe->pipe_state & PIPE_DIRECTW)
1387 			*(int *)data = mpipe->pipe_map.cnt;
1388 		else
1389 			*(int *)data = mpipe->pipe_buffer.cnt;
1390 		break;
1391 
1392 	case FIOSETOWN:
1393 		PIPE_UNLOCK(mpipe);
1394 		error = fsetown(*(int *)data, &mpipe->pipe_sigio);
1395 		goto out_unlocked;
1396 
1397 	case FIOGETOWN:
1398 		*(int *)data = fgetown(&mpipe->pipe_sigio);
1399 		break;
1400 
1401 	/* This is deprecated, FIOSETOWN should be used instead. */
1402 	case TIOCSPGRP:
1403 		PIPE_UNLOCK(mpipe);
1404 		error = fsetown(-(*(int *)data), &mpipe->pipe_sigio);
1405 		goto out_unlocked;
1406 
1407 	/* This is deprecated, FIOGETOWN should be used instead. */
1408 	case TIOCGPGRP:
1409 		*(int *)data = -fgetown(&mpipe->pipe_sigio);
1410 		break;
1411 
1412 	default:
1413 		error = ENOTTY;
1414 		break;
1415 	}
1416 	PIPE_UNLOCK(mpipe);
1417 out_unlocked:
1418 	return (error);
1419 }
1420 
1421 static int
1422 pipe_poll(fp, events, active_cred, td)
1423 	struct file *fp;
1424 	int events;
1425 	struct ucred *active_cred;
1426 	struct thread *td;
1427 {
1428 	struct pipe *rpipe;
1429 	struct pipe *wpipe;
1430 	int levents, revents;
1431 #ifdef MAC
1432 	int error;
1433 #endif
1434 
1435 	revents = 0;
1436 	rpipe = fp->f_data;
1437 	wpipe = PIPE_PEER(rpipe);
1438 	PIPE_LOCK(rpipe);
1439 #ifdef MAC
1440 	error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair);
1441 	if (error)
1442 		goto locked_error;
1443 #endif
1444 	if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM))
1445 		if ((rpipe->pipe_state & PIPE_DIRECTW) ||
1446 		    (rpipe->pipe_buffer.cnt > 0))
1447 			revents |= events & (POLLIN | POLLRDNORM);
1448 
1449 	if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM))
1450 		if (wpipe->pipe_present != PIPE_ACTIVE ||
1451 		    (wpipe->pipe_state & PIPE_EOF) ||
1452 		    (((wpipe->pipe_state & PIPE_DIRECTW) == 0) &&
1453 		     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||
1454 			 wpipe->pipe_buffer.size == 0)))
1455 			revents |= events & (POLLOUT | POLLWRNORM);
1456 
1457 	levents = events &
1458 	    (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND);
1459 	if (rpipe->pipe_state & PIPE_NAMED && fp->f_flag & FREAD && levents &&
1460 	    fp->f_seqcount == rpipe->pipe_wgen)
1461 		events |= POLLINIGNEOF;
1462 
1463 	if ((events & POLLINIGNEOF) == 0) {
1464 		if (rpipe->pipe_state & PIPE_EOF) {
1465 			revents |= (events & (POLLIN | POLLRDNORM));
1466 			if (wpipe->pipe_present != PIPE_ACTIVE ||
1467 			    (wpipe->pipe_state & PIPE_EOF))
1468 				revents |= POLLHUP;
1469 		}
1470 	}
1471 
1472 	if (revents == 0) {
1473 		if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) {
1474 			selrecord(td, &rpipe->pipe_sel);
1475 			if (SEL_WAITING(&rpipe->pipe_sel))
1476 				rpipe->pipe_state |= PIPE_SEL;
1477 		}
1478 
1479 		if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) {
1480 			selrecord(td, &wpipe->pipe_sel);
1481 			if (SEL_WAITING(&wpipe->pipe_sel))
1482 				wpipe->pipe_state |= PIPE_SEL;
1483 		}
1484 	}
1485 #ifdef MAC
1486 locked_error:
1487 #endif
1488 	PIPE_UNLOCK(rpipe);
1489 
1490 	return (revents);
1491 }
1492 
1493 /*
1494  * We shouldn't need locks here as we're doing a read and this should
1495  * be a natural race.
1496  */
1497 static int
1498 pipe_stat(fp, ub, active_cred, td)
1499 	struct file *fp;
1500 	struct stat *ub;
1501 	struct ucred *active_cred;
1502 	struct thread *td;
1503 {
1504 	struct pipe *pipe;
1505 	int new_unr;
1506 #ifdef MAC
1507 	int error;
1508 #endif
1509 
1510 	pipe = fp->f_data;
1511 	PIPE_LOCK(pipe);
1512 #ifdef MAC
1513 	error = mac_pipe_check_stat(active_cred, pipe->pipe_pair);
1514 	if (error) {
1515 		PIPE_UNLOCK(pipe);
1516 		return (error);
1517 	}
1518 #endif
1519 
1520 	/* For named pipes ask the underlying filesystem. */
1521 	if (pipe->pipe_state & PIPE_NAMED) {
1522 		PIPE_UNLOCK(pipe);
1523 		return (vnops.fo_stat(fp, ub, active_cred, td));
1524 	}
1525 
1526 	/*
1527 	 * Lazily allocate an inode number for the pipe.  Most pipe
1528 	 * users do not call fstat(2) on the pipe, which means that
1529 	 * postponing the inode allocation until it is must be
1530 	 * returned to userland is useful.  If alloc_unr failed,
1531 	 * assign st_ino zero instead of returning an error.
1532 	 * Special pipe_ino values:
1533 	 *  -1 - not yet initialized;
1534 	 *  0  - alloc_unr failed, return 0 as st_ino forever.
1535 	 */
1536 	if (pipe->pipe_ino == (ino_t)-1) {
1537 		new_unr = alloc_unr(pipeino_unr);
1538 		if (new_unr != -1)
1539 			pipe->pipe_ino = new_unr;
1540 		else
1541 			pipe->pipe_ino = 0;
1542 	}
1543 	PIPE_UNLOCK(pipe);
1544 
1545 	bzero(ub, sizeof(*ub));
1546 	ub->st_mode = S_IFIFO;
1547 	ub->st_blksize = PAGE_SIZE;
1548 	if (pipe->pipe_state & PIPE_DIRECTW)
1549 		ub->st_size = pipe->pipe_map.cnt;
1550 	else
1551 		ub->st_size = pipe->pipe_buffer.cnt;
1552 	ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize;
1553 	ub->st_atim = pipe->pipe_atime;
1554 	ub->st_mtim = pipe->pipe_mtime;
1555 	ub->st_ctim = pipe->pipe_ctime;
1556 	ub->st_uid = fp->f_cred->cr_uid;
1557 	ub->st_gid = fp->f_cred->cr_gid;
1558 	ub->st_dev = pipedev_ino;
1559 	ub->st_ino = pipe->pipe_ino;
1560 	/*
1561 	 * Left as 0: st_nlink, st_rdev, st_flags, st_gen.
1562 	 */
1563 	return (0);
1564 }
1565 
1566 /* ARGSUSED */
1567 static int
1568 pipe_close(fp, td)
1569 	struct file *fp;
1570 	struct thread *td;
1571 {
1572 
1573 	if (fp->f_vnode != NULL)
1574 		return vnops.fo_close(fp, td);
1575 	fp->f_ops = &badfileops;
1576 	pipe_dtor(fp->f_data);
1577 	fp->f_data = NULL;
1578 	return (0);
1579 }
1580 
1581 static int
1582 pipe_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td)
1583 {
1584 	struct pipe *cpipe;
1585 	int error;
1586 
1587 	cpipe = fp->f_data;
1588 	if (cpipe->pipe_state & PIPE_NAMED)
1589 		error = vn_chmod(fp, mode, active_cred, td);
1590 	else
1591 		error = invfo_chmod(fp, mode, active_cred, td);
1592 	return (error);
1593 }
1594 
1595 static int
1596 pipe_chown(fp, uid, gid, active_cred, td)
1597 	struct file *fp;
1598 	uid_t uid;
1599 	gid_t gid;
1600 	struct ucred *active_cred;
1601 	struct thread *td;
1602 {
1603 	struct pipe *cpipe;
1604 	int error;
1605 
1606 	cpipe = fp->f_data;
1607 	if (cpipe->pipe_state & PIPE_NAMED)
1608 		error = vn_chown(fp, uid, gid, active_cred, td);
1609 	else
1610 		error = invfo_chown(fp, uid, gid, active_cred, td);
1611 	return (error);
1612 }
1613 
1614 static int
1615 pipe_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
1616 {
1617 	struct pipe *pi;
1618 
1619 	if (fp->f_type == DTYPE_FIFO)
1620 		return (vn_fill_kinfo(fp, kif, fdp));
1621 	kif->kf_type = KF_TYPE_PIPE;
1622 	pi = fp->f_data;
1623 	kif->kf_un.kf_pipe.kf_pipe_addr = (uintptr_t)pi;
1624 	kif->kf_un.kf_pipe.kf_pipe_peer = (uintptr_t)pi->pipe_peer;
1625 	kif->kf_un.kf_pipe.kf_pipe_buffer_cnt = pi->pipe_buffer.cnt;
1626 	return (0);
1627 }
1628 
1629 static void
1630 pipe_free_kmem(cpipe)
1631 	struct pipe *cpipe;
1632 {
1633 
1634 	KASSERT(!mtx_owned(PIPE_MTX(cpipe)),
1635 	    ("pipe_free_kmem: pipe mutex locked"));
1636 
1637 	if (cpipe->pipe_buffer.buffer != NULL) {
1638 		atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size);
1639 		vm_map_remove(pipe_map,
1640 		    (vm_offset_t)cpipe->pipe_buffer.buffer,
1641 		    (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size);
1642 		cpipe->pipe_buffer.buffer = NULL;
1643 	}
1644 #ifndef PIPE_NODIRECT
1645 	{
1646 		cpipe->pipe_map.cnt = 0;
1647 		cpipe->pipe_map.pos = 0;
1648 		cpipe->pipe_map.npages = 0;
1649 	}
1650 #endif
1651 }
1652 
1653 /*
1654  * shutdown the pipe
1655  */
1656 static void
1657 pipeclose(cpipe)
1658 	struct pipe *cpipe;
1659 {
1660 	struct pipepair *pp;
1661 	struct pipe *ppipe;
1662 
1663 	KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL"));
1664 
1665 	PIPE_LOCK(cpipe);
1666 	pipelock(cpipe, 0);
1667 	pp = cpipe->pipe_pair;
1668 
1669 	pipeselwakeup(cpipe);
1670 
1671 	/*
1672 	 * If the other side is blocked, wake it up saying that
1673 	 * we want to close it down.
1674 	 */
1675 	cpipe->pipe_state |= PIPE_EOF;
1676 	while (cpipe->pipe_busy) {
1677 		wakeup(cpipe);
1678 		cpipe->pipe_state |= PIPE_WANT;
1679 		pipeunlock(cpipe);
1680 		msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0);
1681 		pipelock(cpipe, 0);
1682 	}
1683 
1684 
1685 	/*
1686 	 * Disconnect from peer, if any.
1687 	 */
1688 	ppipe = cpipe->pipe_peer;
1689 	if (ppipe->pipe_present == PIPE_ACTIVE) {
1690 		pipeselwakeup(ppipe);
1691 
1692 		ppipe->pipe_state |= PIPE_EOF;
1693 		wakeup(ppipe);
1694 		KNOTE_LOCKED(&ppipe->pipe_sel.si_note, 0);
1695 	}
1696 
1697 	/*
1698 	 * Mark this endpoint as free.  Release kmem resources.  We
1699 	 * don't mark this endpoint as unused until we've finished
1700 	 * doing that, or the pipe might disappear out from under
1701 	 * us.
1702 	 */
1703 	PIPE_UNLOCK(cpipe);
1704 	pipe_free_kmem(cpipe);
1705 	PIPE_LOCK(cpipe);
1706 	cpipe->pipe_present = PIPE_CLOSING;
1707 	pipeunlock(cpipe);
1708 
1709 	/*
1710 	 * knlist_clear() may sleep dropping the PIPE_MTX. Set the
1711 	 * PIPE_FINALIZED, that allows other end to free the
1712 	 * pipe_pair, only after the knotes are completely dismantled.
1713 	 */
1714 	knlist_clear(&cpipe->pipe_sel.si_note, 1);
1715 	cpipe->pipe_present = PIPE_FINALIZED;
1716 	seldrain(&cpipe->pipe_sel);
1717 	knlist_destroy(&cpipe->pipe_sel.si_note);
1718 
1719 	/*
1720 	 * If both endpoints are now closed, release the memory for the
1721 	 * pipe pair.  If not, unlock.
1722 	 */
1723 	if (ppipe->pipe_present == PIPE_FINALIZED) {
1724 		PIPE_UNLOCK(cpipe);
1725 #ifdef MAC
1726 		mac_pipe_destroy(pp);
1727 #endif
1728 		uma_zfree(pipe_zone, cpipe->pipe_pair);
1729 	} else
1730 		PIPE_UNLOCK(cpipe);
1731 }
1732 
1733 /*ARGSUSED*/
1734 static int
1735 pipe_kqfilter(struct file *fp, struct knote *kn)
1736 {
1737 	struct pipe *cpipe;
1738 
1739 	/*
1740 	 * If a filter is requested that is not supported by this file
1741 	 * descriptor, don't return an error, but also don't ever generate an
1742 	 * event.
1743 	 */
1744 	if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) {
1745 		kn->kn_fop = &pipe_nfiltops;
1746 		return (0);
1747 	}
1748 	if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) {
1749 		kn->kn_fop = &pipe_nfiltops;
1750 		return (0);
1751 	}
1752 	cpipe = fp->f_data;
1753 	PIPE_LOCK(cpipe);
1754 	switch (kn->kn_filter) {
1755 	case EVFILT_READ:
1756 		kn->kn_fop = &pipe_rfiltops;
1757 		break;
1758 	case EVFILT_WRITE:
1759 		kn->kn_fop = &pipe_wfiltops;
1760 		if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) {
1761 			/* other end of pipe has been closed */
1762 			PIPE_UNLOCK(cpipe);
1763 			return (EPIPE);
1764 		}
1765 		cpipe = PIPE_PEER(cpipe);
1766 		break;
1767 	default:
1768 		PIPE_UNLOCK(cpipe);
1769 		return (EINVAL);
1770 	}
1771 
1772 	kn->kn_hook = cpipe;
1773 	knlist_add(&cpipe->pipe_sel.si_note, kn, 1);
1774 	PIPE_UNLOCK(cpipe);
1775 	return (0);
1776 }
1777 
1778 static void
1779 filt_pipedetach(struct knote *kn)
1780 {
1781 	struct pipe *cpipe = kn->kn_hook;
1782 
1783 	PIPE_LOCK(cpipe);
1784 	knlist_remove(&cpipe->pipe_sel.si_note, kn, 1);
1785 	PIPE_UNLOCK(cpipe);
1786 }
1787 
1788 /*ARGSUSED*/
1789 static int
1790 filt_piperead(struct knote *kn, long hint)
1791 {
1792 	struct pipe *rpipe = kn->kn_hook;
1793 	struct pipe *wpipe = rpipe->pipe_peer;
1794 	int ret;
1795 
1796 	PIPE_LOCK_ASSERT(rpipe, MA_OWNED);
1797 	kn->kn_data = rpipe->pipe_buffer.cnt;
1798 	if ((kn->kn_data == 0) && (rpipe->pipe_state & PIPE_DIRECTW))
1799 		kn->kn_data = rpipe->pipe_map.cnt;
1800 
1801 	if ((rpipe->pipe_state & PIPE_EOF) ||
1802 	    wpipe->pipe_present != PIPE_ACTIVE ||
1803 	    (wpipe->pipe_state & PIPE_EOF)) {
1804 		kn->kn_flags |= EV_EOF;
1805 		return (1);
1806 	}
1807 	ret = kn->kn_data > 0;
1808 	return ret;
1809 }
1810 
1811 /*ARGSUSED*/
1812 static int
1813 filt_pipewrite(struct knote *kn, long hint)
1814 {
1815 	struct pipe *wpipe;
1816 
1817 	wpipe = kn->kn_hook;
1818 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
1819 	if (wpipe->pipe_present != PIPE_ACTIVE ||
1820 	    (wpipe->pipe_state & PIPE_EOF)) {
1821 		kn->kn_data = 0;
1822 		kn->kn_flags |= EV_EOF;
1823 		return (1);
1824 	}
1825 	kn->kn_data = (wpipe->pipe_buffer.size > 0) ?
1826 	    (wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) : PIPE_BUF;
1827 	if (wpipe->pipe_state & PIPE_DIRECTW)
1828 		kn->kn_data = 0;
1829 
1830 	return (kn->kn_data >= PIPE_BUF);
1831 }
1832 
1833 static void
1834 filt_pipedetach_notsup(struct knote *kn)
1835 {
1836 
1837 }
1838 
1839 static int
1840 filt_pipenotsup(struct knote *kn, long hint)
1841 {
1842 
1843 	return (0);
1844 }
1845