242 static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
243 static void pipe_destroy_write_buffer(struct pipe *wpipe);
244 static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
245 static void pipe_clone_write_buffer(struct pipe *wpipe);
300 	struct pipe *rpipe, *wpipe;  in pipe_zone_ctor()  local
316 	wpipe = &pp->pp_wpipe;  in pipe_zone_ctor()
317 	bzero(wpipe, sizeof(*wpipe));  in pipe_zone_ctor()
318 	wpipe->pipe_ctime = rpipe->pipe_ctime;  in pipe_zone_ctor()
319 	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;  in pipe_zone_ctor()
321 	rpipe->pipe_peer = wpipe;  in pipe_zone_ctor()
323 	wpipe->pipe_peer = rpipe;  in pipe_zone_ctor()
324 	wpipe->pipe_pair = pp;  in pipe_zone_ctor()
332 	wpipe->pipe_present = PIPE_ACTIVE;  in pipe_zone_ctor()
373 	struct pipe *rpipe, *wpipe;  in pipe_paircreate()  local
387 	wpipe = &pp->pp_wpipe;  in pipe_paircreate()
391 	knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));  in pipe_paircreate()
400 	error = pipe_create(wpipe, false);  in pipe_paircreate()
416 	wpipe->pipe_state |= PIPE_DIRECTOK;  in pipe_paircreate()
421 	knlist_destroy(&wpipe->pipe_sel.si_note);  in pipe_paircreate()
481 	struct pipe *rpipe, *wpipe;  in kern_pipe()  local
489 	wpipe = &pp->pp_wpipe;  in kern_pipe()
493 		pipeclose(wpipe);  in kern_pipe()
515 		pipeclose(wpipe);  in kern_pipe()
519 	finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);  in kern_pipe()
947 pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)  in pipe_build_write_buffer()  argument
952 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_build_write_buffer()
953 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_build_write_buffer()
954 	    ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));  in pipe_build_write_buffer()
955 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_build_write_buffer()
956 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_build_write_buffer()
958 	if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)  in pipe_build_write_buffer()
959                 size = wpipe->pipe_buffer.size;  in pipe_build_write_buffer()
963 	wpipe->pipe_state |= PIPE_DIRECTW;  in pipe_build_write_buffer()
964 	PIPE_UNLOCK(wpipe);  in pipe_build_write_buffer()
967 	    wpipe->pipe_pages.ms, PIPENPAGES);  in pipe_build_write_buffer()
968 	PIPE_LOCK(wpipe);  in pipe_build_write_buffer()
970 		wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_build_write_buffer()
974 	wpipe->pipe_pages.npages = i;  in pipe_build_write_buffer()
975 	wpipe->pipe_pages.pos =  in pipe_build_write_buffer()
977 	wpipe->pipe_pages.cnt = size;  in pipe_build_write_buffer()
994 pipe_destroy_write_buffer(struct pipe *wpipe)  in pipe_destroy_write_buffer()  argument
997 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_destroy_write_buffer()
998 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_destroy_write_buffer()
999 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_destroy_write_buffer()
1000 	KASSERT(wpipe->pipe_pages.cnt == 0,  in pipe_destroy_write_buffer()
1001 	    ("%s: pipe map for %p contains residual data", __func__, wpipe));  in pipe_destroy_write_buffer()
1003 	wpipe->pipe_state &= ~PIPE_DIRECTW;  in pipe_destroy_write_buffer()
1004 	vm_page_unhold_pages(wpipe->pipe_pages.ms, wpipe->pipe_pages.npages);  in pipe_destroy_write_buffer()
1005 	wpipe->pipe_pages.npages = 0;  in pipe_destroy_write_buffer()
1014 pipe_clone_write_buffer(struct pipe *wpipe)  in pipe_clone_write_buffer()  argument
1021 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_clone_write_buffer()
1022 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,  in pipe_clone_write_buffer()
1023 	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));  in pipe_clone_write_buffer()
1025 	size = wpipe->pipe_pages.cnt;  in pipe_clone_write_buffer()
1026 	pos = wpipe->pipe_pages.pos;  in pipe_clone_write_buffer()
1027 	wpipe->pipe_pages.cnt = 0;  in pipe_clone_write_buffer()
1029 	wpipe->pipe_buffer.in = size;  in pipe_clone_write_buffer()
1030 	wpipe->pipe_buffer.out = 0;  in pipe_clone_write_buffer()
1031 	wpipe->pipe_buffer.cnt = size;  in pipe_clone_write_buffer()
1033 	PIPE_UNLOCK(wpipe);  in pipe_clone_write_buffer()
1034 	iov.iov_base = wpipe->pipe_buffer.buffer;  in pipe_clone_write_buffer()
1043 	uiomove_fromphys(wpipe->pipe_pages.ms, pos, size, &uio);  in pipe_clone_write_buffer()
1044 	PIPE_LOCK(wpipe);  in pipe_clone_write_buffer()
1045 	pipe_destroy_write_buffer(wpipe);  in pipe_clone_write_buffer()
1056 pipe_direct_write(struct pipe *wpipe, struct uio *uio)  in pipe_direct_write()  argument
1061 	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in pipe_direct_write()
1062 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
1066 	if (wpipe->pipe_state & PIPE_DIRECTW) {  in pipe_direct_write()
1067 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1068 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1069 			wakeup(wpipe);  in pipe_direct_write()
1071 		pipeselwakeup(wpipe);  in pipe_direct_write()
1072 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1073 		pipeunlock(wpipe);  in pipe_direct_write()
1074 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
1076 		pipelock(wpipe, false);  in pipe_direct_write()
1081 	if (wpipe->pipe_buffer.cnt > 0) {  in pipe_direct_write()
1082 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1083 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1084 			wakeup(wpipe);  in pipe_direct_write()
1086 		pipeselwakeup(wpipe);  in pipe_direct_write()
1087 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1088 		pipeunlock(wpipe);  in pipe_direct_write()
1089 		error = msleep(wpipe, PIPE_MTX(wpipe),  in pipe_direct_write()
1091 		pipelock(wpipe, false);  in pipe_direct_write()
1097 	error = pipe_build_write_buffer(wpipe, uio);  in pipe_direct_write()
1102 	while (wpipe->pipe_pages.cnt != 0 &&  in pipe_direct_write()
1103 	    (wpipe->pipe_state & PIPE_EOF) == 0) {  in pipe_direct_write()
1104 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_direct_write()
1105 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_direct_write()
1106 			wakeup(wpipe);  in pipe_direct_write()
1108 		pipeselwakeup(wpipe);  in pipe_direct_write()
1109 		wpipe->pipe_state |= PIPE_WANTW;  in pipe_direct_write()
1110 		pipeunlock(wpipe);  in pipe_direct_write()
1111 		error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,  in pipe_direct_write()
1113 		pipelock(wpipe, false);  in pipe_direct_write()
1118 	if ((wpipe->pipe_state & PIPE_EOF) != 0) {  in pipe_direct_write()
1119 		wpipe->pipe_pages.cnt = 0;  in pipe_direct_write()
1120 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1121 		pipeselwakeup(wpipe);  in pipe_direct_write()
1124 		pipe_clone_write_buffer(wpipe);  in pipe_direct_write()
1126 		pipe_destroy_write_buffer(wpipe);  in pipe_direct_write()
1128 	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,  in pipe_direct_write()
1129 	    ("pipe %p leaked PIPE_DIRECTW", wpipe));  in pipe_direct_write()
1133 	wakeup(wpipe);  in pipe_direct_write()
1142 	struct pipe *wpipe, *rpipe;  in pipe_write()  local
1147 	wpipe = PIPE_PEER(rpipe);  in pipe_write()
1149 	error = pipelock(wpipe, true);  in pipe_write()
1157 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_write()
1158 	    (wpipe->pipe_state & PIPE_EOF)) {  in pipe_write()
1159 		pipeunlock(wpipe);  in pipe_write()
1164 	error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);  in pipe_write()
1166 		pipeunlock(wpipe);  in pipe_write()
1171 	++wpipe->pipe_busy;  in pipe_write()
1174 	desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);  in pipe_write()
1175 	while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {  in pipe_write()
1187 	    wpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&  in pipe_write()
1188 	    wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&  in pipe_write()
1193 	if (desiredsize != wpipe->pipe_buffer.size &&  in pipe_write()
1194 	    (wpipe->pipe_state & PIPE_DIRECTW) == 0) {  in pipe_write()
1195 		PIPE_UNLOCK(wpipe);  in pipe_write()
1196 		pipespace(wpipe, desiredsize);  in pipe_write()
1197 		PIPE_LOCK(wpipe);  in pipe_write()
1199 	MPASS(wpipe->pipe_buffer.size != 0);  in pipe_write()
1206 		if (wpipe->pipe_state & PIPE_EOF) {  in pipe_write()
1222 		    wpipe->pipe_buffer.size >= pipe_mindirect &&  in pipe_write()
1224 			error = pipe_direct_write(wpipe, uio);  in pipe_write()
1238 		if (wpipe->pipe_pages.cnt != 0) {  in pipe_write()
1239 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1240 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1241 				wakeup(wpipe);  in pipe_write()
1243 			pipeselwakeup(wpipe);  in pipe_write()
1244 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1245 			pipeunlock(wpipe);  in pipe_write()
1246 			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,  in pipe_write()
1248 			pipelock(wpipe, false);  in pipe_write()
1254 		space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;  in pipe_write()
1279 			segsize = wpipe->pipe_buffer.size -  in pipe_write()
1280 				wpipe->pipe_buffer.in;  in pipe_write()
1287 			error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],  in pipe_write()
1292 				KASSERT(wpipe->pipe_buffer.in + segsize ==  in pipe_write()
1293 					wpipe->pipe_buffer.size,  in pipe_write()
1303 				    &wpipe->pipe_buffer.buffer[0],  in pipe_write()
1308 				wpipe->pipe_buffer.in += size;  in pipe_write()
1309 				if (wpipe->pipe_buffer.in >=  in pipe_write()
1310 				    wpipe->pipe_buffer.size) {  in pipe_write()
1311 					KASSERT(wpipe->pipe_buffer.in ==  in pipe_write()
1313 						wpipe->pipe_buffer.size,  in pipe_write()
1315 					wpipe->pipe_buffer.in = size - segsize;  in pipe_write()
1318 				wpipe->pipe_buffer.cnt += size;  in pipe_write()
1319 				KASSERT(wpipe->pipe_buffer.cnt <=  in pipe_write()
1320 					wpipe->pipe_buffer.size,  in pipe_write()
1330 			if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1331 				wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1332 				wakeup(wpipe);  in pipe_write()
1347 			pipeselwakeup(wpipe);  in pipe_write()
1349 			wpipe->pipe_state |= PIPE_WANTW;  in pipe_write()
1350 			pipeunlock(wpipe);  in pipe_write()
1351 			error = msleep(wpipe, PIPE_MTX(rpipe),  in pipe_write()
1353 			pipelock(wpipe, false);  in pipe_write()
1360 	--wpipe->pipe_busy;  in pipe_write()
1362 	if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {  in pipe_write()
1363 		wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);  in pipe_write()
1364 		wakeup(wpipe);  in pipe_write()
1365 	} else if (wpipe->pipe_buffer.cnt > 0) {  in pipe_write()
1370 		if (wpipe->pipe_state & PIPE_WANTR) {  in pipe_write()
1371 			wpipe->pipe_state &= ~PIPE_WANTR;  in pipe_write()
1372 			wakeup(wpipe);  in pipe_write()
1386 		pipe_timestamp(&wpipe->pipe_mtime);  in pipe_write()
1392 	if (wpipe->pipe_buffer.cnt)  in pipe_write()
1393 		pipeselwakeup(wpipe);  in pipe_write()
1395 	pipeunlock(wpipe);  in pipe_write()
1497 	struct pipe *wpipe;  in pipe_poll()  local
1505 	wpipe = PIPE_PEER(rpipe);  in pipe_poll()
1517 		if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1518 		    (wpipe->pipe_state & PIPE_EOF) ||  in pipe_poll()
1519 		    ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&  in pipe_poll()
1520 		     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||  in pipe_poll()
1521 			 wpipe->pipe_buffer.size == 0)))  in pipe_poll()
1534 			if (wpipe->pipe_present != PIPE_ACTIVE ||  in pipe_poll()
1535 			    (wpipe->pipe_state & PIPE_EOF))  in pipe_poll()
1552 		    wpipe->pipe_present == PIPE_ACTIVE) {  in pipe_poll()
1553 			selrecord(td, &wpipe->pipe_sel);  in pipe_poll()
1554 			if (SEL_WAITING(&wpipe->pipe_sel))  in pipe_poll()
1555 				wpipe->pipe_state |= PIPE_SEL;  in pipe_poll()
1867 	struct pipe *wpipe = kn->kn_hook;  in filt_pipewrite()  local
1873 	if (wpipe->pipe_present == PIPE_ACTIVE ||  in filt_pipewrite()
1874 	    (wpipe->pipe_type & PIPE_TYPE_NAMED) != 0) {  in filt_pipewrite()
1875 		PIPE_LOCK_ASSERT(wpipe, MA_OWNED);  in filt_pipewrite()
1877 		if (wpipe->pipe_state & PIPE_DIRECTW) {  in filt_pipewrite()
1879 		} else if (wpipe->pipe_buffer.size > 0) {  in filt_pipewrite()
1880 			kn->kn_data = wpipe->pipe_buffer.size -  in filt_pipewrite()
1881 			    wpipe->pipe_buffer.cnt;  in filt_pipewrite()
1887 	if (wpipe->pipe_present != PIPE_ACTIVE ||  in filt_pipewrite()
1888 	    (wpipe->pipe_state & PIPE_EOF)) {  in filt_pipewrite()