315 #define	RELE_PROC(xprt) \  argument
316 	((xprt)->xp_type == T_RDMA ? rdma_rele : \
317 	(((xprt)->xp_type == T_CLTS) ? rpc_rele : mir_rele))
325 #define	version_keepquiet(xprt)	(FALSE)  argument
784 svc_xprt_register(SVCMASTERXPRT *xprt, int id)  in svc_xprt_register()  argument
816 	xprt->xp_pool = pool;  in svc_xprt_register()
823 		pool->p_lhead = xprt->xp_prev = xprt->xp_next = xprt;  in svc_xprt_register()
828 		xprt->xp_next = next;  in svc_xprt_register()
829 		xprt->xp_prev = prev;  in svc_xprt_register()
831 		pool->p_lhead = prev->xp_next = next->xp_prev = xprt;  in svc_xprt_register()
847 svc_xprt_unregister(SVCMASTERXPRT *xprt)  in svc_xprt_unregister()  argument
849 	SVCPOOL *pool = xprt->xp_pool;  in svc_xprt_unregister()
859 	if (xprt == xprt->xp_next)  in svc_xprt_unregister()
862 		SVCMASTERXPRT *next = xprt->xp_next;  in svc_xprt_unregister()
863 		SVCMASTERXPRT *prev = xprt->xp_prev;  in svc_xprt_unregister()
868 		if (pool->p_lhead == xprt)  in svc_xprt_unregister()
872 	xprt->xp_next = xprt->xp_prev = NULL;  in svc_xprt_unregister()
919 svc_xprt_qput(SVCPOOL *pool, SVCMASTERXPRT *xprt)  in svc_xprt_qput()  argument
939 	pool->p_qtop->q_xprt = xprt;  in svc_xprt_qput()
956 	SVCMASTERXPRT *xprt;  in svc_xprt_qget()  local
973 		xprt = pool->p_qend->q_xprt;  in svc_xprt_qget()
977 	} while (xprt == NULL);  in svc_xprt_qget()
980 	return (xprt);  in svc_xprt_qget()
989 svc_xprt_qdelete(SVCPOOL *pool, SVCMASTERXPRT *xprt)  in svc_xprt_qdelete()  argument
995 		if (q->q_xprt == xprt)  in svc_xprt_qdelete()
1016 svc_xprt_cleanup(SVCMASTERXPRT *xprt, bool_t detached)  in svc_xprt_cleanup()  argument
1018 	ASSERT(MUTEX_HELD(&xprt->xp_thread_lock));  in svc_xprt_cleanup()
1019 	ASSERT(xprt->xp_wq == NULL);  in svc_xprt_cleanup()
1025 	if (!detached && xprt->xp_threads == 0 && xprt->xp_closeproc) {  in svc_xprt_cleanup()
1026 		(*(xprt->xp_closeproc)) (xprt);  in svc_xprt_cleanup()
1029 	if (xprt->xp_threads + xprt->xp_detached_threads > 0)  in svc_xprt_cleanup()
1030 		mutex_exit(&xprt->xp_thread_lock);  in svc_xprt_cleanup()
1033 		svc_xprt_qdelete(xprt->xp_pool, xprt);  in svc_xprt_cleanup()
1036 		svc_xprt_unregister(xprt);  in svc_xprt_cleanup()
1037 		svc_callout_free(xprt);  in svc_xprt_cleanup()
1038 		SVC_DESTROY(xprt);  in svc_xprt_cleanup()
1053 svc_callout_find(SVCXPRT *xprt, rpcprog_t prog, rpcvers_t vers,  in svc_callout_find()  argument
1056 	SVC_CALLOUT_TABLE *sct = xprt->xp_sct;  in svc_callout_find()
1084 svc_callout_free(SVCMASTERXPRT *xprt)  in svc_callout_free()  argument
1086 	SVC_CALLOUT_TABLE *sct = xprt->xp_sct;  in svc_callout_free()
1467 svc_clone_link(SVCMASTERXPRT *xprt, SVCXPRT *clone_xprt, SVCXPRT *clone_xprt2)  in svc_clone_link()  argument
1478 	mutex_enter(&xprt->xp_thread_lock);  in svc_clone_link()
1479 	xprt->xp_threads++;  in svc_clone_link()
1480 	mutex_exit(&xprt->xp_thread_lock);  in svc_clone_link()
1486 	clone_xprt->xp_master = xprt;  in svc_clone_link()
1489 	clone_xprt->xp_xpc = xprt->xp_xpc;  in svc_clone_link()
1510 	SVCMASTERXPRT *xprt = clone_xprt->xp_master;  in svc_clone_unlink()  local
1514 	ASSERT(xprt->xp_threads > 0);  in svc_clone_unlink()
1517 	mutex_enter(&xprt->xp_thread_lock);  in svc_clone_unlink()
1518 	xprt->xp_threads--;  in svc_clone_unlink()
1521 	if (xprt->xp_wq)  in svc_clone_unlink()
1522 		mutex_exit(&xprt->xp_thread_lock);  in svc_clone_unlink()
1524 		svc_xprt_cleanup(xprt, FALSE);  in svc_clone_unlink()
1545 	SVCMASTERXPRT *xprt = clone_xprt->xp_master;  in svc_clone_unlinkdetached()  local
1549 	ASSERT(xprt->xp_detached_threads > 0);  in svc_clone_unlinkdetached()
1550 	ASSERT(xprt->xp_threads + xprt->xp_detached_threads > 0);  in svc_clone_unlinkdetached()
1553 	mutex_enter(&xprt->xp_thread_lock);  in svc_clone_unlinkdetached()
1554 	xprt->xp_detached_threads--;  in svc_clone_unlinkdetached()
1557 	if (xprt->xp_wq)  in svc_clone_unlinkdetached()
1558 		mutex_exit(&xprt->xp_thread_lock);  in svc_clone_unlinkdetached()
1560 		svc_xprt_cleanup(xprt, TRUE);  in svc_clone_unlinkdetached()
1840 svc_poll(SVCPOOL *pool, SVCMASTERXPRT *xprt, SVCXPRT *clone_xprt)  in svc_poll()  argument
1869 		if (xprt && xprt->xp_req_head && (!pool->p_qoverflow ||  in svc_poll()
1871 			mutex_enter(&xprt->xp_req_lock);  in svc_poll()
1872 			if (xprt->xp_req_head)  in svc_poll()
1873 				return (xprt);  in svc_poll()
1874 			mutex_exit(&xprt->xp_req_lock);  in svc_poll()
1955 		if (xprt == NULL && pool->p_lhead == NULL) {  in svc_poll()
1965 		next = xprt ? xprt->xp_next : pool->p_lhead;  in svc_poll()
2061 		if (xprt && xprt->xp_wq == NULL)  in svc_poll()
2156 svc_flowcontrol(SVCMASTERXPRT *xprt)  in svc_flowcontrol()  argument
2158 	SVCPOOL *pool = xprt->xp_pool;  in svc_flowcontrol()
2162 	ASSERT(MUTEX_HELD(&xprt->xp_req_lock));  in svc_flowcontrol()
2165 	if (xprt->xp_full == FALSE) {  in svc_flowcontrol()
2171 		if (xprt->xp_reqs >= enough_reqs * 2) {  in svc_flowcontrol()
2172 			xprt->xp_full = TRUE;  in svc_flowcontrol()
2181 		    xprt->xp_size >= totalmem / 5 / pool->p_lcount)  in svc_flowcontrol()
2182 			xprt->xp_full = TRUE;  in svc_flowcontrol()
2190 	if (xprt->xp_reqs > enough_reqs)  in svc_flowcontrol()
2198 	    xprt->xp_size >= totalmem / 5 / pool->p_lcount / 2)  in svc_flowcontrol()
2202 	xprt->xp_full = FALSE;  in svc_flowcontrol()
2203 	xprt->xp_enable = TRUE;  in svc_flowcontrol()
2218 	SVCMASTERXPRT *xprt = NULL;	/* master transport handle  */  in svc_run()  local
2248 		next = svc_poll(pool, xprt, clone_xprt);  in svc_run()
2265 			xprt = NULL;  in svc_run()
2317 		if (next != xprt) {  in svc_run()
2318 			if (xprt)  in svc_run()
2321 			xprt = next;  in svc_run()
2385 		mutex_enter(&xprt->xp_req_lock);  in svc_run()
2386 		enable = xprt->xp_enable;  in svc_run()
2388 			xprt->xp_enable = FALSE;  in svc_run()
2389 		mutex_exit(&xprt->xp_req_lock);  in svc_run()
2390 		(*RELE_PROC(xprt)) (clone_xprt->xp_wq, NULL, enable);  in svc_run()
2402 	SVCMASTERXPRT *xprt = ((void **) q->q_ptr)[0];  in svc_queueclean()  local
2409 	mutex_enter(&xprt->xp_req_lock);  in svc_queueclean()
2410 	pool = xprt->xp_pool;  in svc_queueclean()
2411 	while ((mp = xprt->xp_req_head) != NULL) {  in svc_queueclean()
2413 		xprt->xp_req_head = mp->b_next;  in svc_queueclean()
2415 		(*RELE_PROC(xprt)) (xprt->xp_wq, mp, FALSE);  in svc_queueclean()
2419 	pool->p_reqs -= xprt->xp_reqs;  in svc_queueclean()
2420 	pool->p_size -= xprt->xp_size;  in svc_queueclean()
2423 	xprt->xp_reqs = 0;  in svc_queueclean()
2424 	xprt->xp_size = 0;  in svc_queueclean()
2425 	xprt->xp_full = FALSE;  in svc_queueclean()
2426 	xprt->xp_enable = FALSE;  in svc_queueclean()
2427 	mutex_exit(&xprt->xp_req_lock);  in svc_queueclean()
2445 	SVCMASTERXPRT *xprt = ((void **) q->q_ptr)[0];  in svc_queueclose()  local
2447 	if (xprt == NULL) {  in svc_queueclose()
2457 	mutex_enter(&xprt->xp_thread_lock);  in svc_queueclose()
2459 	ASSERT(xprt->xp_req_head == NULL);  in svc_queueclose()
2460 	ASSERT(xprt->xp_wq != NULL);  in svc_queueclose()
2462 	xprt->xp_wq = NULL;  in svc_queueclose()
2464 	if (xprt->xp_threads == 0) {  in svc_queueclose()
2465 		SVCPOOL *pool = xprt->xp_pool;  in svc_queueclose()
2471 		svc_xprt_cleanup(xprt, FALSE);  in svc_queueclose()
2502 		mutex_exit(&xprt->xp_thread_lock);  in svc_queueclose()
2518 	SVCMASTERXPRT *xprt = ((void **) q->q_ptr)[0];  in svc_queuereq()  local
2519 	SVCPOOL *pool = xprt->xp_pool;  in svc_queuereq()
2536 	mutex_enter(&xprt->xp_req_lock);  in svc_queuereq()
2537 	if (flowcontrol && xprt->xp_full) {  in svc_queuereq()
2538 		mutex_exit(&xprt->xp_req_lock);  in svc_queuereq()
2542 	ASSERT(xprt->xp_full == FALSE);  in svc_queuereq()
2544 	if (xprt->xp_req_head == NULL)  in svc_queuereq()
2545 		xprt->xp_req_head = mp;  in svc_queuereq()
2547 		xprt->xp_req_tail->b_next = mp;  in svc_queuereq()
2548 	xprt->xp_req_tail = mp;  in svc_queuereq()
2558 	svc_xprt_qput(pool, xprt);  in svc_queuereq()
2562 	xprt->xp_reqs++;  in svc_queuereq()
2565 	xprt->xp_size += size;  in svc_queuereq()
2570 		svc_flowcontrol(xprt);  in svc_queuereq()
2596 	mutex_exit(&xprt->xp_req_lock);  in svc_queuereq()
2705 	SVCMASTERXPRT *xprt = clone_xprt->xp_master;  in svc_detach_thread()  local
2706 	SVCPOOL *pool = xprt->xp_pool;  in svc_detach_thread()
2714 	mutex_enter(&xprt->xp_thread_lock);  in svc_detach_thread()
2715 	xprt->xp_threads--;  in svc_detach_thread()
2716 	xprt->xp_detached_threads++;  in svc_detach_thread()
2717 	mutex_exit(&xprt->xp_thread_lock);  in svc_detach_thread()
2727 	mutex_enter(&xprt->xp_req_lock);  in svc_detach_thread()
2728 	enable = xprt->xp_enable;  in svc_detach_thread()
2730 		xprt->xp_enable = FALSE;  in svc_detach_thread()
2731 	mutex_exit(&xprt->xp_req_lock);  in svc_detach_thread()
2732 	(*RELE_PROC(xprt)) (clone_xprt->xp_wq, NULL, enable);  in svc_detach_thread()
2753 	SVCMASTERXPRT *xprt;  in rdma_stop()  local
2770 		xprt = curr_rec->rtr_xprt_ptr;  in rdma_stop()
2771 		q = xprt->xp_wq;  in rdma_stop()
2772 		svc_rdma_kstop(xprt);  in rdma_stop()
2774 		mutex_enter(&xprt->xp_req_lock);  in rdma_stop()
2775 		pool = xprt->xp_pool;  in rdma_stop()
2776 		while ((mp = xprt->xp_req_head) != NULL) {  in rdma_stop()
2780 			xprt->xp_req_head = mp->b_next;  in rdma_stop()
2788 		pool->p_reqs -= xprt->xp_reqs;  in rdma_stop()
2789 		pool->p_size -= xprt->xp_size;  in rdma_stop()
2791 		xprt->xp_reqs = 0;  in rdma_stop()
2792 		xprt->xp_size = 0;  in rdma_stop()
2793 		xprt->xp_full = FALSE;  in rdma_stop()
2794 		xprt->xp_enable = FALSE;  in rdma_stop()
2795 		mutex_exit(&xprt->xp_req_lock);  in rdma_stop()