106 	isc__taskmgr_t *		manager;  member
242 empty_readyq(isc__taskmgr_t *manager);
245 pop_readyq(isc__taskmgr_t *manager);
248 push_readyq(isc__taskmgr_t *manager, isc__task_t *task);
298 	isc__taskmgr_t *manager = task->manager;  in task_finished()  local
307 	LOCK(&manager->lock);  in task_finished()
308 	UNLINK(manager->tasks, task, link);  in task_finished()
310 	if (FINISHED(manager)) {  in task_finished()
317 		BROADCAST(&manager->work_available);  in task_finished()
320 	UNLOCK(&manager->lock);  in task_finished()
325 	isc_mem_put(manager->mctx, task, sizeof(*task));  in task_finished()
332 	isc__taskmgr_t *manager = (void*)manager0;  in isc__task_create()  local
337 	REQUIRE(VALID_MANAGER(manager));  in isc__task_create()
340 	task = isc_mem_get(manager->mctx, sizeof(*task));  in isc__task_create()
346 		isc_mem_put(manager->mctx, task, sizeof(*task));  in isc__task_create()
349 	LOCK(&manager->lock);  in isc__task_create()
351 	task->manager = manager;  in isc__task_create()
365 	UNLOCK(&manager->lock);  in isc__task_create()
368 	LOCK(&manager->lock);  in isc__task_create()
369 	if (!manager->exiting) {  in isc__task_create()
371 			task->quantum = manager->default_quantum;  in isc__task_create()
372 		APPEND(manager->tasks, task, link);  in isc__task_create()
375 	UNLOCK(&manager->lock);  in isc__task_create()
379 		isc_mem_put(manager->mctx, task, sizeof(*task));  in isc__task_create()
456 	isc__taskmgr_t *manager = task->manager;  in task_ready()  local
461 	REQUIRE(VALID_MANAGER(manager));  in task_ready()
466 	LOCK(&manager->lock);  in task_ready()
467 	push_readyq(manager, task);  in task_ready()
469 	if (manager->mode == isc_taskmgrmode_normal || has_privilege)  in task_ready()
470 		SIGNAL(&manager->work_available);  in task_ready()
472 	UNLOCK(&manager->lock);  in task_ready()
811 	event = isc_event_allocate(task->manager->mctx,  in isc__task_onshutdown()
829 		isc_mem_put(task->manager->mctx, event, sizeof(*event));  in isc__task_onshutdown()
925 empty_readyq(isc__taskmgr_t *manager) {  in empty_readyq()  argument
928 	if (manager->mode == isc_taskmgrmode_normal)  in empty_readyq()
929 		queue = manager->ready_tasks;  in empty_readyq()
931 		queue = manager->ready_priority_tasks;  in empty_readyq()
945 pop_readyq(isc__taskmgr_t *manager) {  in pop_readyq()  argument
948 	if (manager->mode == isc_taskmgrmode_normal)  in pop_readyq()
949 		task = HEAD(manager->ready_tasks);  in pop_readyq()
951 		task = HEAD(manager->ready_priority_tasks);  in pop_readyq()
954 		DEQUEUE(manager->ready_tasks, task, ready_link);  in pop_readyq()
956 			DEQUEUE(manager->ready_priority_tasks, task,  in pop_readyq()
970 push_readyq(isc__taskmgr_t *manager, isc__task_t *task) {  in push_readyq()  argument
971 	ENQUEUE(manager->ready_tasks, task, ready_link);  in push_readyq()
973 		ENQUEUE(manager->ready_priority_tasks, task,  in push_readyq()
978 dispatch(isc__taskmgr_t *manager) {  in dispatch()  argument
986 	REQUIRE(VALID_MANAGER(manager));  in dispatch()
1042 	LOCK(&manager->lock);  in dispatch()
1044 	while (!FINISHED(manager)) {  in dispatch()
1056 		while ((empty_readyq(manager) || manager->pause_requested ||  in dispatch()
1057 			manager->exclusive_requested) && !FINISHED(manager))  in dispatch()
1062 			WAIT(&manager->work_available, &manager->lock);  in dispatch()
1069 		    empty_readyq(manager))  in dispatch()
1075 		task = pop_readyq(manager);  in dispatch()
1090 			manager->tasks_running++;  in dispatch()
1091 			UNLOCK(&manager->lock);  in dispatch()
1204 			LOCK(&manager->lock);  in dispatch()
1205 			manager->tasks_running--;  in dispatch()
1207 			if (manager->exclusive_requested &&  in dispatch()
1208 			    manager->tasks_running == 1) {  in dispatch()
1209 				SIGNAL(&manager->exclusive_granted);  in dispatch()
1210 			} else if (manager->pause_requested &&  in dispatch()
1211 				   manager->tasks_running == 0) {  in dispatch()
1212 				SIGNAL(&manager->paused);  in dispatch()
1236 				push_readyq(manager, task);  in dispatch()
1253 		if (manager->tasks_running == 0 && empty_readyq(manager)) {  in dispatch()
1254 			manager->mode = isc_taskmgrmode_normal;  in dispatch()
1255 			if (!empty_readyq(manager))  in dispatch()
1256 				BROADCAST(&manager->work_available);  in dispatch()
1262 	ISC_LIST_APPENDLIST(manager->ready_tasks, new_ready_tasks, ready_link);  in dispatch()
1263 	ISC_LIST_APPENDLIST(manager->ready_priority_tasks, new_priority_tasks,  in dispatch()
1265 	if (empty_readyq(manager))  in dispatch()
1266 		manager->mode = isc_taskmgrmode_normal;  in dispatch()
1269 	UNLOCK(&manager->lock);  in dispatch()
1278 	isc__taskmgr_t *manager = uap;  in run()  local
1283 	dispatch(manager);  in run()
1297 manager_free(isc__taskmgr_t *manager) {  in manager_free()  argument
1300 	LOCK(&manager->lock);  in manager_free()
1302 	(void)isc_condition_destroy(&manager->exclusive_granted);  in manager_free()
1303 	(void)isc_condition_destroy(&manager->work_available);  in manager_free()
1304 	(void)isc_condition_destroy(&manager->paused);  in manager_free()
1305 	isc_mem_free(manager->mctx, manager->threads);  in manager_free()
1307 	manager->common.impmagic = 0;  in manager_free()
1308 	manager->common.magic = 0;  in manager_free()
1309 	mctx = manager->mctx;  in manager_free()
1310 	UNLOCK(&manager->lock);  in manager_free()
1311 	DESTROYLOCK(&manager->lock);  in manager_free()
1312 	isc_mem_put(mctx, manager, sizeof(*manager));  in manager_free()
1326 	isc__taskmgr_t *manager;  in isc__taskmgr_create()  local
1350 	manager = isc_mem_get(mctx, sizeof(*manager));  in isc__taskmgr_create()
1351 	if (manager == NULL)  in isc__taskmgr_create()
1353 	manager->common.methods = &taskmgrmethods;  in isc__taskmgr_create()
1354 	manager->common.impmagic = TASK_MANAGER_MAGIC;  in isc__taskmgr_create()
1355 	manager->common.magic = ISCAPI_TASKMGR_MAGIC;  in isc__taskmgr_create()
1356 	manager->mode = isc_taskmgrmode_normal;  in isc__taskmgr_create()
1357 	manager->mctx = NULL;  in isc__taskmgr_create()
1358 	result = isc_mutex_init(&manager->lock);  in isc__taskmgr_create()
1361 	LOCK(&manager->lock);  in isc__taskmgr_create()
1364 	manager->workers = 0;  in isc__taskmgr_create()
1365 	manager->threads = isc_mem_allocate(mctx,  in isc__taskmgr_create()
1367 	if (manager->threads == NULL) {  in isc__taskmgr_create()
1371 	if (isc_condition_init(&manager->work_available) != ISC_R_SUCCESS) {  in isc__taskmgr_create()
1379 	if (isc_condition_init(&manager->exclusive_granted) != ISC_R_SUCCESS) {  in isc__taskmgr_create()
1387 	if (isc_condition_init(&manager->paused) != ISC_R_SUCCESS) {  in isc__taskmgr_create()
1398 	manager->default_quantum = default_quantum;  in isc__taskmgr_create()
1399 	INIT_LIST(manager->tasks);  in isc__taskmgr_create()
1400 	INIT_LIST(manager->ready_tasks);  in isc__taskmgr_create()
1401 	INIT_LIST(manager->ready_priority_tasks);  in isc__taskmgr_create()
1402 	manager->tasks_running = 0;  in isc__taskmgr_create()
1403 	manager->exclusive_requested = ISC_FALSE;  in isc__taskmgr_create()
1404 	manager->pause_requested = ISC_FALSE;  in isc__taskmgr_create()
1405 	manager->exiting = ISC_FALSE;  in isc__taskmgr_create()
1407 	isc_mem_attach(mctx, &manager->mctx);  in isc__taskmgr_create()
1414 		if (isc_thread_create(run, manager,  in isc__taskmgr_create()
1415 				      &manager->threads[manager->workers]) ==  in isc__taskmgr_create()
1417 			manager->workers++;  in isc__taskmgr_create()
1421 	UNLOCK(&manager->lock);  in isc__taskmgr_create()
1424 		manager_free(manager);  in isc__taskmgr_create()
1430 	manager->refs = 1;  in isc__taskmgr_create()
1431 	UNLOCK(&manager->lock);  in isc__taskmgr_create()
1432 	taskmgr = manager;  in isc__taskmgr_create()
1435 	*managerp = (isc_taskmgr_t *)manager;  in isc__taskmgr_create()
1441 	(void)isc_condition_destroy(&manager->exclusive_granted);  in isc__taskmgr_create()
1443 	(void)isc_condition_destroy(&manager->work_available);  in isc__taskmgr_create()
1445 	isc_mem_free(mctx, manager->threads);  in isc__taskmgr_create()
1447 	UNLOCK(&manager->lock);  in isc__taskmgr_create()
1448 	DESTROYLOCK(&manager->lock);  in isc__taskmgr_create()
1451 	isc_mem_put(mctx, manager, sizeof(*manager));  in isc__taskmgr_create()
1457 	isc__taskmgr_t *manager;  in isc__taskmgr_destroy()  local
1466 	manager = (void*)(*managerp);  in isc__taskmgr_destroy()
1467 	REQUIRE(VALID_MANAGER(manager));  in isc__taskmgr_destroy()
1474 	manager->refs--;  in isc__taskmgr_destroy()
1475 	if (manager->refs > 0) {  in isc__taskmgr_destroy()
1499 	LOCK(&manager->lock);  in isc__taskmgr_destroy()
1504 	INSIST(!manager->exiting);  in isc__taskmgr_destroy()
1505 	manager->exiting = ISC_TRUE;  in isc__taskmgr_destroy()
1510 	manager->mode = isc_taskmgrmode_normal;  in isc__taskmgr_destroy()
1516 	for (task = HEAD(manager->tasks);  in isc__taskmgr_destroy()
1521 			push_readyq(manager, task);  in isc__taskmgr_destroy()
1530 	BROADCAST(&manager->work_available);  in isc__taskmgr_destroy()
1531 	UNLOCK(&manager->lock);  in isc__taskmgr_destroy()
1536 	for (i = 0; i < manager->workers; i++)  in isc__taskmgr_destroy()
1537 		(void)isc_thread_join(manager->threads[i], NULL);  in isc__taskmgr_destroy()
1542 	UNLOCK(&manager->lock);  in isc__taskmgr_destroy()
1543 	while (isc__taskmgr_ready((isc_taskmgr_t *)manager))  in isc__taskmgr_destroy()
1544 		(void)isc__taskmgr_dispatch((isc_taskmgr_t *)manager);  in isc__taskmgr_destroy()
1546 	if (!ISC_LIST_EMPTY(manager->tasks))  in isc__taskmgr_destroy()
1549 	INSIST(ISC_LIST_EMPTY(manager->tasks));  in isc__taskmgr_destroy()
1555 	manager_free(manager);  in isc__taskmgr_destroy()
1562 	isc__taskmgr_t *manager = (void*)manager0;  in isc__taskmgr_setmode()  local
1564 	LOCK(&manager->lock);  in isc__taskmgr_setmode()
1565 	manager->mode = mode;  in isc__taskmgr_setmode()
1566 	UNLOCK(&manager->lock);  in isc__taskmgr_setmode()
1571 	isc__taskmgr_t *manager = (void*)manager0;  in isc__taskmgr_mode()  local
1573 	LOCK(&manager->lock);  in isc__taskmgr_mode()
1574 	mode = manager->mode;  in isc__taskmgr_mode()
1575 	UNLOCK(&manager->lock);  in isc__taskmgr_mode()
1582 	isc__taskmgr_t *manager = (void*)manager0;  in isc__taskmgr_ready()  local
1586 	if (manager == NULL)  in isc__taskmgr_ready()
1587 		manager = taskmgr;  in isc__taskmgr_ready()
1589 	if (manager == NULL)  in isc__taskmgr_ready()
1592 	LOCK(&manager->lock);  in isc__taskmgr_ready()
1593 	is_ready = !empty_readyq(manager);  in isc__taskmgr_ready()
1594 	UNLOCK(&manager->lock);  in isc__taskmgr_ready()
1601 	isc__taskmgr_t *manager = (void*)manager0;  in isc__taskmgr_dispatch()  local
1604 	if (manager == NULL)  in isc__taskmgr_dispatch()
1605 		manager = taskmgr;  in isc__taskmgr_dispatch()
1607 	if (manager == NULL)  in isc__taskmgr_dispatch()
1610 	dispatch(manager);  in isc__taskmgr_dispatch()
1618 	isc__taskmgr_t *manager = (void*)manager0;  in isc__taskmgr_pause()  local
1619 	LOCK(&manager->lock);  in isc__taskmgr_pause()
1620 	while (manager->tasks_running > 0) {  in isc__taskmgr_pause()
1621 		WAIT(&manager->paused, &manager->lock);  in isc__taskmgr_pause()
1623 	manager->pause_requested = ISC_TRUE;  in isc__taskmgr_pause()
1624 	UNLOCK(&manager->lock);  in isc__taskmgr_pause()
1629 	isc__taskmgr_t *manager = (void*)manager0;  in isc__taskmgr_resume()  local
1631 	LOCK(&manager->lock);  in isc__taskmgr_resume()
1632 	if (manager->pause_requested) {  in isc__taskmgr_resume()
1633 		manager->pause_requested = ISC_FALSE;  in isc__taskmgr_resume()
1634 		BROADCAST(&manager->work_available);  in isc__taskmgr_resume()
1636 	UNLOCK(&manager->lock);  in isc__taskmgr_resume()
1644 	isc__taskmgr_t *manager = task->manager;  in isc__task_beginexclusive()  local
1646 	LOCK(&manager->lock);  in isc__task_beginexclusive()
1647 	if (manager->exclusive_requested) {  in isc__task_beginexclusive()
1648 		UNLOCK(&manager->lock);  in isc__task_beginexclusive()
1651 	manager->exclusive_requested = ISC_TRUE;  in isc__task_beginexclusive()
1652 	while (manager->tasks_running > 1) {  in isc__task_beginexclusive()
1653 		WAIT(&manager->exclusive_granted, &manager->lock);  in isc__task_beginexclusive()
1655 	UNLOCK(&manager->lock);  in isc__task_beginexclusive()
1666 	isc__taskmgr_t *manager = task->manager;  in isc__task_endexclusive()  local
1669 	LOCK(&manager->lock);  in isc__task_endexclusive()
1670 	REQUIRE(manager->exclusive_requested);  in isc__task_endexclusive()
1671 	manager->exclusive_requested = ISC_FALSE;  in isc__task_endexclusive()
1672 	BROADCAST(&manager->work_available);  in isc__task_endexclusive()
1673 	UNLOCK(&manager->lock);  in isc__task_endexclusive()
1682 	isc__taskmgr_t *manager = task->manager;  in isc__task_setprivilege()  local
1696 	LOCK(&manager->lock);  in isc__task_setprivilege()
1698 		ENQUEUE(manager->ready_priority_tasks, task,  in isc__task_setprivilege()
1701 		DEQUEUE(manager->ready_priority_tasks, task,  in isc__task_setprivilege()
1703 	UNLOCK(&manager->lock);  in isc__task_setprivilege()