80 	aio_req_t *reqp = (aio_req_t *)aio->aio_private;  in aphysio()  local
81 	struct buf *bp = &reqp->aio_req_buf;  in aphysio()
128 		bp->b_forw = (struct buf *)reqp;  in aphysio()
149 	reqp->aio_req_flags |= AIO_PAGELOCKDONE;  in aphysio()
159 	reqp->aio_req_cancel = cancel;  in aphysio()
188 	aio_req_t *reqp;  in aio_done()  local
206 	reqp = (aio_req_t *)bp->b_forw;  in aio_done()
207 	fd = reqp->aio_req_fd;  in aio_done()
225 	ASSERT(reqp->aio_req_flags & AIO_PENDING);  in aio_done()
227 	reqp->aio_req_flags &= ~AIO_PENDING;  in aio_done()
228 	reqp_flags = reqp->aio_req_flags;  in aio_done()
229 	if ((pkevp = reqp->aio_req_portkev) != NULL) {  in aio_done()
231 		if (reqp->aio_req_flags & AIO_CLOSE_PORT) {  in aio_done()
237 			aio_deq(&aiop->aio_portpending, reqp);  in aio_done()
238 			aio_enq(&aiop->aio_portq, reqp, 0);  in aio_done()
253 			aio_deq(&aiop->aio_portpending, reqp);  in aio_done()
254 			aio_enq(&aiop->aio_portcleanupq, reqp, 0);  in aio_done()
262 		aio_deq(&aiop->aio_portpending, reqp);  in aio_done()
263 		aio_enq(&aiop->aio_portq, reqp, 0);  in aio_done()
274 		pollqflag = (reqp->aio_req_flags & AIO_POLL);  in aio_done()
289 				aio_enq(&aiop->aio_pollq, reqp, AIO_POLLQ);  in aio_done()
290 			else if (reqp->aio_req_sigqp || reqp->aio_req_lio)  in aio_done()
291 				aio_enq(&aiop->aio_notifyq, reqp, AIO_NOTIFYQ);  in aio_done()
293 				aio_enq(&aiop->aio_cleanupq, reqp,  in aio_done()
331 		sigev = reqp->aio_req_sigqp;  in aio_done()
332 		reqp->aio_req_sigqp = NULL;  in aio_done()
335 		aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);  in aio_done()
342 	if ((head = reqp->aio_req_lio) != NULL) {  in aio_done()
380 	    (reqp->aio_req_flags & AIO_SOLARIS) &&  in aio_done()
384 		reqp->aio_req_flags |= AIO_SIGNALLED;  in aio_done()
440 aio_zerolen(aio_req_t *reqp)  in aio_zerolen()  argument
443 	struct buf *bp = &reqp->aio_req_buf;  in aio_zerolen()
445 	reqp->aio_req_flags |= AIO_ZEROLEN;  in aio_zerolen()
447 	bp->b_forw = (struct buf *)reqp;  in aio_zerolen()
460 aphysio_unlock(aio_req_t *reqp)  in aphysio_unlock()  argument
466 	if (reqp->aio_req_flags & AIO_PHYSIODONE)  in aphysio_unlock()
469 	reqp->aio_req_flags |= AIO_PHYSIODONE;  in aphysio_unlock()
471 	if (reqp->aio_req_flags & AIO_ZEROLEN)  in aphysio_unlock()
474 	bp = &reqp->aio_req_buf;  in aphysio_unlock()
475 	iov = reqp->aio_req_uio.uio_iov;  in aphysio_unlock()
477 	if (reqp->aio_req_flags & AIO_PAGELOCKDONE) {  in aphysio_unlock()
481 		reqp->aio_req_flags &= ~AIO_PAGELOCKDONE;  in aphysio_unlock()
491 aio_hash_delete(aio_t *aiop, struct aio_req_t *reqp)  in aio_hash_delete()  argument
494 	aio_result_t *resultp = reqp->aio_req_resultp;  in aio_hash_delete()
527 aio_req_free(aio_t *aiop, aio_req_t *reqp)  in aio_req_free()  argument
533 	if (reqp->aio_req_portkev) {  in aio_req_free()
534 		port_free_event(reqp->aio_req_portkev);  in aio_req_free()
535 		reqp->aio_req_portkev = NULL;  in aio_req_free()
538 	if ((liop = reqp->aio_req_lio) != NULL) {  in aio_req_free()
541 		reqp->aio_req_lio = NULL;  in aio_req_free()
543 	if (reqp->aio_req_sigqp != NULL) {  in aio_req_free()
544 		kmem_free(reqp->aio_req_sigqp, sizeof (sigqueue_t));  in aio_req_free()
545 		reqp->aio_req_sigqp = NULL;  in aio_req_free()
547 	reqp->aio_req_next = aiop->aio_free;  in aio_req_free()
548 	reqp->aio_req_prev = NULL;  in aio_req_free()
549 	aiop->aio_free = reqp;  in aio_req_free()
553 	aio_hash_delete(aiop, reqp);  in aio_req_free()
560 aio_req_free_port(aio_t *aiop, aio_req_t *reqp)  in aio_req_free_port()  argument
564 	reqp->aio_req_next = aiop->aio_free;  in aio_req_free_port()
565 	reqp->aio_req_prev = NULL;  in aio_req_free_port()
566 	aiop->aio_free = reqp;  in aio_req_free_port()
568 	aio_hash_delete(aiop, reqp);  in aio_req_free_port()
580 	aio_req_t *reqp;  in aio_verify_queue()  local
584 	if ((reqp = head) != NULL) {  in aio_verify_queue()
586 			ASSERT(reqp->aio_req_prev->aio_req_next == reqp);  in aio_verify_queue()
587 			ASSERT(reqp->aio_req_next->aio_req_prev == reqp);  in aio_verify_queue()
588 			if (entry_present == reqp)  in aio_verify_queue()
590 			if (entry_missing == reqp)  in aio_verify_queue()
592 		} while ((reqp = reqp->aio_req_next) != head);  in aio_verify_queue()
605 aio_enq(aio_req_t **qhead, aio_req_t *reqp, int qflg_new)  in aio_enq()  argument
610 	aio_verify_queue(*qhead, NULL, reqp);  in aio_enq()
613 		reqp->aio_req_next = reqp;  in aio_enq()
614 		reqp->aio_req_prev = reqp;  in aio_enq()
615 		*qhead = reqp;  in aio_enq()
617 		reqp->aio_req_next = head;  in aio_enq()
618 		reqp->aio_req_prev = prev = head->aio_req_prev;  in aio_enq()
619 		prev->aio_req_next = reqp;  in aio_enq()
620 		head->aio_req_prev = reqp;  in aio_enq()
622 	reqp->aio_req_flags |= qflg_new;  in aio_enq()
629 aio_deq(aio_req_t **qhead, aio_req_t *reqp)  in aio_deq()  argument
631 	aio_verify_queue(*qhead, reqp, NULL);  in aio_deq()
633 	if (reqp->aio_req_next == reqp) {  in aio_deq()
636 		reqp->aio_req_prev->aio_req_next = reqp->aio_req_next;  in aio_deq()
637 		reqp->aio_req_next->aio_req_prev = reqp->aio_req_prev;  in aio_deq()
638 		if (*qhead == reqp)  in aio_deq()
639 			*qhead = reqp->aio_req_next;  in aio_deq()
641 	reqp->aio_req_next = NULL;  in aio_deq()
642 	reqp->aio_req_prev = NULL;  in aio_deq()
655 	aio_req_t *reqp = q2;  in aio_cleanupq_concat()  local
658 		ASSERT(reqp->aio_req_flags & qflg);  in aio_cleanupq_concat()
659 		reqp->aio_req_flags &= ~qflg;  in aio_cleanupq_concat()
660 		reqp->aio_req_flags |= AIO_CLEANUPQ;  in aio_cleanupq_concat()
661 	} while ((reqp = reqp->aio_req_next) != q2);  in aio_cleanupq_concat()
792 	aio_req_t	*reqp;  in aio_cleanup_portq()  local
815 		if ((reqp = headp) != NULL) {  in aio_cleanup_portq()
817 				next = reqp->aio_req_next;  in aio_cleanup_portq()
818 				aphysio_unlock(reqp);  in aio_cleanup_portq()
821 					aio_req_free(aiop, reqp);  in aio_cleanup_portq()
824 			} while ((reqp = next) != headp);  in aio_cleanup_portq()
848 	if ((reqp = cleanupq) == NULL)  in aio_cleanup_portq()
851 		next = reqp->aio_req_next;  in aio_cleanup_portq()
852 		aphysio_unlock(reqp);  in aio_cleanup_portq()
855 			aio_req_free(aiop, reqp);  in aio_cleanup_portq()
859 			aio_enq(&aiop->aio_portq, reqp, 0);  in aio_cleanup_portq()
861 			port_send_event(reqp->aio_req_portkev);  in aio_cleanup_portq()
862 			if ((liop = reqp->aio_req_lio) != NULL) {  in aio_cleanup_portq()
879 	} while ((reqp = next) != cleanupq);  in aio_cleanup_portq()
888 	aio_req_t *reqp, *next;  in aio_cleanup_cleanupq()  local
900 	if ((reqp = qhead) == NULL)  in aio_cleanup_cleanupq()
903 		ASSERT(reqp->aio_req_flags & AIO_CLEANUPQ);  in aio_cleanup_cleanupq()
904 		ASSERT(reqp->aio_req_portkev == NULL);  in aio_cleanup_cleanupq()
905 		next = reqp->aio_req_next;  in aio_cleanup_cleanupq()
906 		aphysio_unlock(reqp);  in aio_cleanup_cleanupq()
909 			aio_req_free(aiop, reqp);  in aio_cleanup_cleanupq()
911 			aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);  in aio_cleanup_cleanupq()
913 			if (reqp->aio_req_flags & AIO_SIGNALLED)  in aio_cleanup_cleanupq()
916 				reqp->aio_req_flags |= AIO_SIGNALLED;  in aio_cleanup_cleanupq()
919 	} while ((reqp = next) != qhead);  in aio_cleanup_cleanupq()
929 	aio_req_t *reqp, *next;  in aio_cleanup_notifyq()  local
934 	if ((reqp = qhead) == NULL)  in aio_cleanup_notifyq()
937 		ASSERT(reqp->aio_req_flags & AIO_NOTIFYQ);  in aio_cleanup_notifyq()
938 		next = reqp->aio_req_next;  in aio_cleanup_notifyq()
939 		aphysio_unlock(reqp);  in aio_cleanup_notifyq()
942 			aio_req_free(aiop, reqp);  in aio_cleanup_notifyq()
946 			aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);  in aio_cleanup_notifyq()
947 			sigev = reqp->aio_req_sigqp;  in aio_cleanup_notifyq()
948 			reqp->aio_req_sigqp = NULL;  in aio_cleanup_notifyq()
949 			if ((liohead = reqp->aio_req_lio) != NULL) {  in aio_cleanup_notifyq()
960 				aio_sigev_send(reqp->aio_req_buf.b_proc,  in aio_cleanup_notifyq()
965 				aio_sigev_send(reqp->aio_req_buf.b_proc,  in aio_cleanup_notifyq()
969 	} while ((reqp = next) != qhead);  in aio_cleanup_notifyq()
980 	aio_req_t *reqp, *next;  in aio_cleanup_pollq()  local
986 	if ((reqp = qhead) == NULL)  in aio_cleanup_pollq()
989 		ASSERT(reqp->aio_req_flags & AIO_POLLQ);  in aio_cleanup_pollq()
990 		next = reqp->aio_req_next;  in aio_cleanup_pollq()
991 		aphysio_unlock(reqp);  in aio_cleanup_pollq()
994 			aio_req_free(aiop, reqp);  in aio_cleanup_pollq()
997 			aio_copyout_result(reqp);  in aio_cleanup_pollq()
999 			aio_enq(&aiop->aio_doneq, reqp, AIO_DONEQ);  in aio_cleanup_pollq()
1002 	} while ((reqp = next) != qhead);  in aio_cleanup_pollq()
1014 	aio_req_t *reqp, *next, *head;  in aio_cleanup_exit()  local
1043 		reqp = head;  in aio_cleanup_exit()
1045 			next = reqp->aio_req_next;  in aio_cleanup_exit()
1046 			aphysio_unlock(reqp);  in aio_cleanup_exit()
1047 			kmem_free(reqp, sizeof (struct aio_req_t));  in aio_cleanup_exit()
1048 		} while ((reqp = next) != head);  in aio_cleanup_exit()
1053 	for (reqp = aiop->aio_free; reqp != NULL; reqp = next) {  in aio_cleanup_exit()
1054 		next = reqp->aio_req_next;  in aio_cleanup_exit()
1055 		kmem_free(reqp, sizeof (struct aio_req_t));  in aio_cleanup_exit()
1081 aio_copyout_result(aio_req_t *reqp)  in aio_copyout_result()  argument
1089 	if (reqp->aio_req_flags & AIO_COPYOUTDONE)  in aio_copyout_result()
1092 	reqp->aio_req_flags |= AIO_COPYOUTDONE;  in aio_copyout_result()
1094 	iov = reqp->aio_req_uio.uio_iov;  in aio_copyout_result()
1095 	bp = &reqp->aio_req_buf;  in aio_copyout_result()
1097 	resultp = (void *)reqp->aio_req_resultp;  in aio_copyout_result()
1160 aio_req_remove_portq(aio_t *aiop, aio_req_t *reqp)  in aio_req_remove_portq()  argument
1174 	aio_deq(&aiop->aio_portq, reqp);  in aio_req_remove_portq()
1182 	aio_req_t 	*reqp;  in aio_close_port()  local
1215 		reqp = headp;  in aio_close_port()
1217 			if (reqp->aio_req_portkev &&  in aio_close_port()
1218 			    reqp->aio_req_port == port) {  in aio_close_port()
1219 				reqp->aio_req_flags |= AIO_CLOSE_PORT;  in aio_close_port()
1222 		} while ((reqp = reqp->aio_req_next) != headp);  in aio_close_port()
1240 	if ((reqp = aiop->aio_portq) != NULL) {  in aio_close_port()
1242 			next = reqp->aio_req_next;  in aio_close_port()
1243 			if (reqp->aio_req_port == port) {  in aio_close_port()
1245 				aio_req_remove_portq(aiop, reqp);  in aio_close_port()
1246 				port_free_event(reqp->aio_req_portkev);  in aio_close_port()
1248 				reqp->aio_req_next = headp;  in aio_close_port()
1249 				headp = reqp;  in aio_close_port()
1251 		} while ((reqp = next) != aiop->aio_portq);  in aio_close_port()
1256 	for (reqp = headp; reqp != NULL; reqp = next) {  in aio_close_port()
1257 		next = reqp->aio_req_next;  in aio_close_port()
1258 		aphysio_unlock(reqp);  in aio_close_port()
1260 		aio_req_free_port(aiop, reqp);  in aio_close_port()