300 	struct pipe *rpipe, *wpipe;  in pipe_zone_ctor()  local
311 	rpipe = &pp->pp_rpipe;  in pipe_zone_ctor()
312 	bzero(rpipe, sizeof(*rpipe));  in pipe_zone_ctor()
313 	pipe_timestamp(&rpipe->pipe_ctime);  in pipe_zone_ctor()
314 	rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime;  in pipe_zone_ctor()
318 	wpipe->pipe_ctime = rpipe->pipe_ctime;  in pipe_zone_ctor()
319 	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;  in pipe_zone_ctor()
321 	rpipe->pipe_peer = wpipe;  in pipe_zone_ctor()
322 	rpipe->pipe_pair = pp;  in pipe_zone_ctor()
323 	wpipe->pipe_peer = rpipe;  in pipe_zone_ctor()
331 	rpipe->pipe_present = PIPE_ACTIVE;  in pipe_zone_ctor()
373 	struct pipe *rpipe, *wpipe;  in pipe_paircreate()  local
386 	rpipe = &pp->pp_rpipe;  in pipe_paircreate()
390 	knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe));  in pipe_paircreate()
397 	error = pipe_create(rpipe, true);  in pipe_paircreate()
411 		pipe_free_kmem(rpipe);  in pipe_paircreate()
415 	rpipe->pipe_state |= PIPE_DIRECTOK;  in pipe_paircreate()
420 	knlist_destroy(&rpipe->pipe_sel.si_note);  in pipe_paircreate()
481 	struct pipe *rpipe, *wpipe;  in kern_pipe()  local
488 	rpipe = &pp->pp_rpipe;  in kern_pipe()
492 		pipeclose(rpipe);  in kern_pipe()
509 	finit(rf, fflags, DTYPE_PIPE, rpipe, &pipeops);  in kern_pipe()
751 	struct pipe *rpipe;  in pipe_read()  local
756 	rpipe = fp->f_data;  in pipe_read()
770 		if ((atomic_load_short(&rpipe->pipe_state) & PIPE_EOF) == 0 &&  in pipe_read()
771 		    atomic_load_int(&rpipe->pipe_buffer.cnt) == 0 &&  in pipe_read()
772 		    atomic_load_int(&rpipe->pipe_pages.cnt) == 0)  in pipe_read()
776 	PIPE_LOCK(rpipe);  in pipe_read()
777 	++rpipe->pipe_busy;  in pipe_read()
778 	error = pipelock(rpipe, true);  in pipe_read()
783 	error = mac_pipe_check_read(active_cred, rpipe->pipe_pair);  in pipe_read()
788 		if ((rpipe->pipe_state & PIPE_DIRECTW) == 0 &&  in pipe_read()
789 		    rpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&  in pipe_read()
790 		    rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&  in pipe_read()
792 			PIPE_UNLOCK(rpipe);  in pipe_read()
793 			pipespace(rpipe, SMALL_PIPE_SIZE);  in pipe_read()
794 			PIPE_LOCK(rpipe);  in pipe_read()
802 		if (rpipe->pipe_buffer.cnt > 0) {  in pipe_read()
803 			size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;  in pipe_read()
804 			if (size > rpipe->pipe_buffer.cnt)  in pipe_read()
805 				size = rpipe->pipe_buffer.cnt;  in pipe_read()
809 			PIPE_UNLOCK(rpipe);  in pipe_read()
811 			    &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],  in pipe_read()
813 			PIPE_LOCK(rpipe);  in pipe_read()
817 			rpipe->pipe_buffer.out += size;  in pipe_read()
818 			if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)  in pipe_read()
819 				rpipe->pipe_buffer.out = 0;  in pipe_read()
821 			rpipe->pipe_buffer.cnt -= size;  in pipe_read()
828 			if (rpipe->pipe_buffer.cnt == 0) {  in pipe_read()
829 				rpipe->pipe_buffer.in = 0;  in pipe_read()
830 				rpipe->pipe_buffer.out = 0;  in pipe_read()
837 		} else if ((size = rpipe->pipe_pages.cnt) != 0) {  in pipe_read()
840 			PIPE_UNLOCK(rpipe);  in pipe_read()
841 			error = uiomove_fromphys(rpipe->pipe_pages.ms,  in pipe_read()
842 			    rpipe->pipe_pages.pos, size, uio);  in pipe_read()
843 			PIPE_LOCK(rpipe);  in pipe_read()
847 			rpipe->pipe_pages.pos += size;  in pipe_read()
848 			rpipe->pipe_pages.cnt -= size;  in pipe_read()
849 			if (rpipe->pipe_pages.cnt == 0) {  in pipe_read()
850 				rpipe->pipe_state &= ~PIPE_WANTW;  in pipe_read()
851 				wakeup(rpipe);  in pipe_read()
859 			if (rpipe->pipe_state & PIPE_EOF)  in pipe_read()
865 			if (rpipe->pipe_state & PIPE_WANTW) {  in pipe_read()
866 				rpipe->pipe_state &= ~PIPE_WANTW;  in pipe_read()
867 				wakeup(rpipe);  in pipe_read()
881 			pipeunlock(rpipe);  in pipe_read()
890 				rpipe->pipe_state |= PIPE_WANTR;  in pipe_read()
891 				if ((error = msleep(rpipe, PIPE_MTX(rpipe),  in pipe_read()
894 					error = pipelock(rpipe, true);  in pipe_read()
903 	pipeunlock(rpipe);  in pipe_read()
907 		pipe_timestamp(&rpipe->pipe_atime);  in pipe_read()
909 	--rpipe->pipe_busy;  in pipe_read()
914 	if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {  in pipe_read()
915 		rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);  in pipe_read()
916 		wakeup(rpipe);  in pipe_read()
917 	} else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {  in pipe_read()
921 		if (rpipe->pipe_state & PIPE_WANTW) {  in pipe_read()
922 			rpipe->pipe_state &= ~PIPE_WANTW;  in pipe_read()
923 			wakeup(rpipe);  in pipe_read()
932 	    rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt >= PIPE_BUF)  in pipe_read()
933 		pipeselwakeup(rpipe);  in pipe_read()
935 	PIPE_UNLOCK(rpipe);  in pipe_read()
1142 	struct pipe *wpipe, *rpipe;  in pipe_write()  local
1146 	rpipe = fp->f_data;  in pipe_write()
1147 	wpipe = PIPE_PEER(rpipe);  in pipe_write()
1148 	PIPE_LOCK(rpipe);  in pipe_write()
1151 		PIPE_UNLOCK(rpipe);  in pipe_write()
1160 		PIPE_UNLOCK(rpipe);  in pipe_write()
1167 		PIPE_UNLOCK(rpipe);  in pipe_write()
1246 			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,  in pipe_write()
1286 			PIPE_UNLOCK(rpipe);  in pipe_write()
1289 			PIPE_LOCK(rpipe);  in pipe_write()
1301 				PIPE_UNLOCK(rpipe);  in pipe_write()
1305 				PIPE_LOCK(rpipe);  in pipe_write()
1351 			error = msleep(wpipe, PIPE_MTX(rpipe),  in pipe_write()
1396 	PIPE_UNLOCK(rpipe);  in pipe_write()
1496 	struct pipe *rpipe;  in pipe_poll()  local
1504 	rpipe = fp->f_data;  in pipe_poll()
1505 	wpipe = PIPE_PEER(rpipe);  in pipe_poll()
1506 	PIPE_LOCK(rpipe);  in pipe_poll()
1508 	error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair);  in pipe_poll()
1513 		if (rpipe->pipe_pages.cnt > 0 || rpipe->pipe_buffer.cnt > 0)  in pipe_poll()
1526 	if (rpipe->pipe_type & PIPE_TYPE_NAMED && fp->f_flag & FREAD && levents &&  in pipe_poll()
1527 	    fp->f_pipegen == rpipe->pipe_wgen)  in pipe_poll()
1531 		if (rpipe->pipe_state & PIPE_EOF) {  in pipe_poll()
1546 			selrecord(td, &rpipe->pipe_sel);  in pipe_poll()
1547 			if (SEL_WAITING(&rpipe->pipe_sel))  in pipe_poll()
1548 				rpipe->pipe_state |= PIPE_SEL;  in pipe_poll()
1561 	PIPE_UNLOCK(rpipe);  in pipe_poll()
1846 	struct pipe *rpipe = kn->kn_hook;  in filt_piperead()  local
1848 	PIPE_LOCK_ASSERT(rpipe, MA_OWNED);  in filt_piperead()
1849 	kn->kn_data = rpipe->pipe_buffer.cnt;  in filt_piperead()
1851 		kn->kn_data = rpipe->pipe_pages.cnt;  in filt_piperead()
1853 	if ((rpipe->pipe_state & PIPE_EOF) != 0 &&  in filt_piperead()
1854 	    ((rpipe->pipe_type & PIPE_TYPE_NAMED) == 0 ||  in filt_piperead()
1855 	    fp->f_pipegen != rpipe->pipe_wgen)) {  in filt_piperead()