72 static struct svc_callout *svc_find(SVCPOOL *pool, rpcprog_t, rpcvers_t,
76 static void svc_change_space_used(SVCPOOL *pool, long delta);
77 static bool_t svc_request_space_available(SVCPOOL *pool);
78 static void svcpool_cleanup(SVCPOOL *pool);
89 	SVCPOOL *pool;  in svcpool_create()  local
93 	pool = malloc(sizeof(SVCPOOL), M_RPC, M_WAITOK|M_ZERO);  in svcpool_create()
95 	mtx_init(&pool->sp_lock, "sp_lock", NULL, MTX_DEF);  in svcpool_create()
96 	pool->sp_name = name;  in svcpool_create()
97 	pool->sp_state = SVCPOOL_INIT;  in svcpool_create()
98 	pool->sp_proc = NULL;  in svcpool_create()
99 	TAILQ_INIT(&pool->sp_callouts);  in svcpool_create()
100 	TAILQ_INIT(&pool->sp_lcallouts);  in svcpool_create()
101 	pool->sp_minthreads = 1;  in svcpool_create()
102 	pool->sp_maxthreads = 1;  in svcpool_create()
103 	pool->sp_groupcount = 1;  in svcpool_create()
105 		grp = &pool->sp_groups[g];  in svcpool_create()
107 		grp->sg_pool = pool;  in svcpool_create()
123 	pool->sp_space_high = (u_long)nmbclusters * MCLBYTES / 4;  in svcpool_create()
124 	pool->sp_space_low = (pool->sp_space_high / 3) * 2;  in svcpool_create()
126 	sysctl_ctx_init(&pool->sp_sysctl);  in svcpool_create()
128 		SYSCTL_ADD_PROC(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
130 		    pool, 0, svcpool_minthread_sysctl, "I",  in svcpool_create()
132 		SYSCTL_ADD_PROC(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
134 		    pool, 0, svcpool_maxthread_sysctl, "I",  in svcpool_create()
136 		SYSCTL_ADD_PROC(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
138 		    pool, 0, svcpool_threads_sysctl, "I",  in svcpool_create()
140 		SYSCTL_ADD_INT(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
141 		    "groups", CTLFLAG_RD, &pool->sp_groupcount, 0,  in svcpool_create()
144 		SYSCTL_ADD_ULONG(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
146 		    &pool->sp_space_used,  in svcpool_create()
149 		SYSCTL_ADD_ULONG(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
151 		    &pool->sp_space_used_highest,  in svcpool_create()
154 		SYSCTL_ADD_ULONG(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
156 		    &pool->sp_space_high,  in svcpool_create()
159 		SYSCTL_ADD_ULONG(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
161 		    &pool->sp_space_low,  in svcpool_create()
164 		SYSCTL_ADD_INT(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
166 		    &pool->sp_space_throttled, 0,  in svcpool_create()
169 		SYSCTL_ADD_INT(&pool->sp_sysctl, sysctl_base, OID_AUTO,  in svcpool_create()
171 		    &pool->sp_space_throttle_count, 0,  in svcpool_create()
175 	return pool;  in svcpool_create()
183 svcpool_cleanup(SVCPOOL *pool)  in svcpool_cleanup()  argument
195 		grp = &pool->sp_groups[g];  in svcpool_cleanup()
209 	mtx_lock(&pool->sp_lock);  in svcpool_cleanup()
210 	while ((s = TAILQ_FIRST(&pool->sp_callouts)) != NULL) {  in svcpool_cleanup()
211 		mtx_unlock(&pool->sp_lock);  in svcpool_cleanup()
212 		svc_unreg(pool, s->sc_prog, s->sc_vers);  in svcpool_cleanup()
213 		mtx_lock(&pool->sp_lock);  in svcpool_cleanup()
215 	while ((sl = TAILQ_FIRST(&pool->sp_lcallouts)) != NULL) {  in svcpool_cleanup()
216 		mtx_unlock(&pool->sp_lock);  in svcpool_cleanup()
217 		svc_loss_unreg(pool, sl->slc_dispatch);  in svcpool_cleanup()
218 		mtx_lock(&pool->sp_lock);  in svcpool_cleanup()
220 	mtx_unlock(&pool->sp_lock);  in svcpool_cleanup()
224 svcpool_destroy(SVCPOOL *pool)  in svcpool_destroy()  argument
229 	svcpool_cleanup(pool);  in svcpool_destroy()
232 		grp = &pool->sp_groups[g];  in svcpool_destroy()
235 	mtx_destroy(&pool->sp_lock);  in svcpool_destroy()
237 	if (pool->sp_rcache)  in svcpool_destroy()
238 		replay_freecache(pool->sp_rcache);  in svcpool_destroy()
240 	sysctl_ctx_free(&pool->sp_sysctl);  in svcpool_destroy()
241 	free(pool, M_RPC);  in svcpool_destroy()
249 svcpool_close(SVCPOOL *pool)  in svcpool_close()  argument
254 	svcpool_cleanup(pool);  in svcpool_close()
257 	mtx_lock(&pool->sp_lock);  in svcpool_close()
258 	pool->sp_state = SVCPOOL_INIT;  in svcpool_close()
259 	mtx_unlock(&pool->sp_lock);  in svcpool_close()
261 		grp = &pool->sp_groups[g];  in svcpool_close()
274 	SVCPOOL *pool;  in svcpool_threads_sysctl()  local
277 	pool = oidp->oid_arg1;  in svcpool_threads_sysctl()
279 	mtx_lock(&pool->sp_lock);  in svcpool_threads_sysctl()
280 	for (g = 0; g < pool->sp_groupcount; g++)  in svcpool_threads_sysctl()
281 		threads += pool->sp_groups[g].sg_threadcount;  in svcpool_threads_sysctl()
282 	mtx_unlock(&pool->sp_lock);  in svcpool_threads_sysctl()
293 	SVCPOOL *pool;  in svcpool_minthread_sysctl()  local
296 	pool = oidp->oid_arg1;  in svcpool_minthread_sysctl()
297 	newminthreads = pool->sp_minthreads;  in svcpool_minthread_sysctl()
299 	if (error == 0 && newminthreads != pool->sp_minthreads) {  in svcpool_minthread_sysctl()
300 		if (newminthreads > pool->sp_maxthreads)  in svcpool_minthread_sysctl()
302 		mtx_lock(&pool->sp_lock);  in svcpool_minthread_sysctl()
303 		pool->sp_minthreads = newminthreads;  in svcpool_minthread_sysctl()
304 		for (g = 0; g < pool->sp_groupcount; g++) {  in svcpool_minthread_sysctl()
305 			pool->sp_groups[g].sg_minthreads = max(1,  in svcpool_minthread_sysctl()
306 			    pool->sp_minthreads / pool->sp_groupcount);  in svcpool_minthread_sysctl()
308 		mtx_unlock(&pool->sp_lock);  in svcpool_minthread_sysctl()
319 	SVCPOOL *pool;  in svcpool_maxthread_sysctl()  local
322 	pool = oidp->oid_arg1;  in svcpool_maxthread_sysctl()
323 	newmaxthreads = pool->sp_maxthreads;  in svcpool_maxthread_sysctl()
325 	if (error == 0 && newmaxthreads != pool->sp_maxthreads) {  in svcpool_maxthread_sysctl()
326 		if (newmaxthreads < pool->sp_minthreads)  in svcpool_maxthread_sysctl()
328 		mtx_lock(&pool->sp_lock);  in svcpool_maxthread_sysctl()
329 		pool->sp_maxthreads = newmaxthreads;  in svcpool_maxthread_sysctl()
330 		for (g = 0; g < pool->sp_groupcount; g++) {  in svcpool_maxthread_sysctl()
331 			pool->sp_groups[g].sg_maxthreads = max(1,  in svcpool_maxthread_sysctl()
332 			    pool->sp_maxthreads / pool->sp_groupcount);  in svcpool_maxthread_sysctl()
334 		mtx_unlock(&pool->sp_lock);  in svcpool_maxthread_sysctl()
345 	SVCPOOL *pool = xprt->xp_pool;  in xprt_register()  local
350 	g = atomic_fetchadd_int(&pool->sp_nextgroup, 1) % pool->sp_groupcount;  in xprt_register()
351 	xprt->xp_group = grp = &pool->sp_groups[g];  in xprt_register()
506 	SVCPOOL *pool = xprt->xp_pool;  in svc_reg()  local
524 	mtx_lock(&pool->sp_lock);  in svc_reg()
525 	if ((s = svc_find(pool, prog, vers, netid)) != NULL) {  in svc_reg()
530 		mtx_unlock(&pool->sp_lock);  in svc_reg()
537 		mtx_unlock(&pool->sp_lock);  in svc_reg()
545 	TAILQ_INSERT_TAIL(&pool->sp_callouts, s, sc_link);  in svc_reg()
551 	mtx_unlock(&pool->sp_lock);  in svc_reg()
570 svc_unreg(SVCPOOL *pool, const rpcprog_t prog, const rpcvers_t vers)  in svc_unreg()  argument
576 	mtx_lock(&pool->sp_lock);  in svc_unreg()
577 	while ((s = svc_find(pool, prog, vers, NULL)) != NULL) {  in svc_unreg()
578 		TAILQ_REMOVE(&pool->sp_callouts, s, sc_link);  in svc_unreg()
583 	mtx_unlock(&pool->sp_lock);  in svc_unreg()
593 	SVCPOOL *pool = xprt->xp_pool;  in svc_loss_reg()  local
596 	mtx_lock(&pool->sp_lock);  in svc_loss_reg()
597 	TAILQ_FOREACH(s, &pool->sp_lcallouts, slc_link) {  in svc_loss_reg()
602 		mtx_unlock(&pool->sp_lock);  in svc_loss_reg()
607 		mtx_unlock(&pool->sp_lock);  in svc_loss_reg()
611 	TAILQ_INSERT_TAIL(&pool->sp_lcallouts, s, slc_link);  in svc_loss_reg()
612 	mtx_unlock(&pool->sp_lock);  in svc_loss_reg()
620 svc_loss_unreg(SVCPOOL *pool, void (*dispatch)(SVCXPRT *))  in svc_loss_unreg()  argument
624 	mtx_lock(&pool->sp_lock);  in svc_loss_unreg()
625 	TAILQ_FOREACH(s, &pool->sp_lcallouts, slc_link) {  in svc_loss_unreg()
627 			TAILQ_REMOVE(&pool->sp_lcallouts, s, slc_link);  in svc_loss_unreg()
632 	mtx_unlock(&pool->sp_lock);  in svc_loss_unreg()
642 svc_find(SVCPOOL *pool, rpcprog_t prog, rpcvers_t vers, char *netid)  in svc_find()  argument
646 	mtx_assert(&pool->sp_lock, MA_OWNED);  in svc_find()
647 	TAILQ_FOREACH(s, &pool->sp_callouts, sc_link) {  in svc_find()
922 	SVCPOOL *pool = xprt->xp_pool;  in svc_getreq()  local
944 		if (pool->sp_rcache) {  in svc_getreq()
948 			rs = replay_find(pool->sp_rcache, &msg,  in svc_getreq()
1023 		TAILQ_FOREACH(s, &pool->sp_lcallouts, slc_link)  in svc_getreq()
1035 	SVCPOOL *pool = xprt->xp_pool;  in svc_executereq()  local
1045 	TAILQ_FOREACH(s, &pool->sp_callouts, sc_link) {  in svc_executereq()
1108 svc_assign_waiting_sockets(SVCPOOL *pool)  in svc_assign_waiting_sockets()  argument
1114 	for (g = 0; g < pool->sp_groupcount; g++) {  in svc_assign_waiting_sockets()
1115 		grp = &pool->sp_groups[g];  in svc_assign_waiting_sockets()
1128 svc_change_space_used(SVCPOOL *pool, long delta)  in svc_change_space_used()  argument
1132 	value = atomic_fetchadd_long(&pool->sp_space_used, delta) + delta;  in svc_change_space_used()
1134 		if (value >= pool->sp_space_high && !pool->sp_space_throttled) {  in svc_change_space_used()
1135 			pool->sp_space_throttled = TRUE;  in svc_change_space_used()
1136 			pool->sp_space_throttle_count++;  in svc_change_space_used()
1138 		if (value > pool->sp_space_used_highest)  in svc_change_space_used()
1139 			pool->sp_space_used_highest = value;  in svc_change_space_used()
1141 		if (value < pool->sp_space_low && pool->sp_space_throttled) {  in svc_change_space_used()
1142 			pool->sp_space_throttled = FALSE;  in svc_change_space_used()
1143 			svc_assign_waiting_sockets(pool);  in svc_change_space_used()
1149 svc_request_space_available(SVCPOOL *pool)  in svc_request_space_available()  argument
1152 	if (pool->sp_space_throttled)  in svc_request_space_available()
1160 	SVCPOOL *pool = grp->sg_pool;  in svc_run_internal()  local
1171 	st->st_pool = pool;  in svc_run_internal()
1220 			if (svc_request_space_available(pool) &&  in svc_run_internal()
1262 					svc_exit(pool);  in svc_run_internal()
1276 			if (!svc_request_space_available(pool))  in svc_run_internal()
1281 				svc_change_space_used(pool, rqstp->rq_size);  in svc_run_internal()
1286 				if (pool->sp_assign) {  in svc_run_internal()
1287 					stpref = pool->sp_assign(st, rqstp);  in svc_run_internal()
1313 			if (!svc_request_space_available(pool) ||  in svc_run_internal()
1330 			svc_change_space_used(pool, -sz);  in svc_run_internal()
1364 	SVCPOOL *pool = grp->sg_pool;  in svc_new_thread()  local
1370 	kthread_add(svc_thread_start, grp, pool->sp_proc, &td, 0, 0,  in svc_new_thread()
1371 	    "%s: service", pool->sp_name);  in svc_new_thread()
1375 svc_run(SVCPOOL *pool)  in svc_run()  argument
1385 	    "%s: master", pool->sp_name);  in svc_run()
1386 	pool->sp_state = SVCPOOL_ACTIVE;  in svc_run()
1387 	pool->sp_proc = p;  in svc_run()
1390 	pool->sp_groupcount = max(1, min(SVC_MAXGROUPS,  in svc_run()
1391 	    min(pool->sp_maxthreads / 2, mp_ncpus) / 6));  in svc_run()
1392 	for (g = 0; g < pool->sp_groupcount; g++) {  in svc_run()
1393 		grp = &pool->sp_groups[g];  in svc_run()
1395 		    pool->sp_minthreads / pool->sp_groupcount);  in svc_run()
1397 		    pool->sp_maxthreads / pool->sp_groupcount);  in svc_run()
1402 	pool->sp_groups[0].sg_threadcount++;  in svc_run()
1403 	for (g = 0; g < pool->sp_groupcount; g++) {  in svc_run()
1404 		grp = &pool->sp_groups[g];  in svc_run()
1408 	svc_run_internal(&pool->sp_groups[0], TRUE);  in svc_run()
1411 	for (g = 0; g < pool->sp_groupcount; g++) {  in svc_run()
1412 		grp = &pool->sp_groups[g];  in svc_run()
1421 svc_exit(SVCPOOL *pool)  in svc_exit()  argument
1427 	pool->sp_state = SVCPOOL_CLOSING;  in svc_exit()
1428 	for (g = 0; g < pool->sp_groupcount; g++) {  in svc_exit()
1429 		grp = &pool->sp_groups[g];  in svc_exit()
1475 	SVCPOOL *pool;  in svc_freereq()  local
1479 		pool = st->st_pool;  in svc_freereq()
1480 		if (pool->sp_done)  in svc_freereq()
1481 			pool->sp_done(st, rqstp);  in svc_freereq()