Lines Matching refs:tq
700 #define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag) \ argument
707 #define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag) \ argument
710 (!(tq->tq_flags & TASKQ_PREPOPULATE) || \
711 (tq->tq_nalloc > tq->tq_minalloc)) && \
713 mutex_exit(&tq->tq_lock); \
717 #define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag) argument
718 #define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag) argument
746 #define TQ_DO_ENQUEUE(tq, tqe, func, arg, front) { \ argument
747 ASSERT(MUTEX_HELD(&tq->tq_lock)); \
750 TQ_PREPEND(tq->tq_task, tqe); \
752 TQ_APPEND(tq->tq_task, tqe); \
756 tq->tq_tasks++; \
757 if (tq->tq_tasks - tq->tq_executed > tq->tq_maxtasks) \
758 tq->tq_maxtasks = tq->tq_tasks - tq->tq_executed; \
759 cv_signal(&tq->tq_dispatch_cv); \
760 DTRACE_PROBE2(taskq__enqueue, taskq_t *, tq, taskq_ent_t *, tqe); \
763 #define TQ_ENQUEUE(tq, tqe, func, arg) \ argument
764 TQ_DO_ENQUEUE(tq, tqe, func, arg, 0)
766 #define TQ_ENQUEUE_FRONT(tq, tqe, func, arg) \ argument
767 TQ_DO_ENQUEUE(tq, tqe, func, arg, 1)
782 taskq_t *tq = buf; in taskq_constructor() local
784 bzero(tq, sizeof (taskq_t)); in taskq_constructor()
786 mutex_init(&tq->tq_lock, NULL, MUTEX_DEFAULT, NULL); in taskq_constructor()
787 rw_init(&tq->tq_threadlock, NULL, RW_DEFAULT, NULL); in taskq_constructor()
788 cv_init(&tq->tq_dispatch_cv, NULL, CV_DEFAULT, NULL); in taskq_constructor()
789 cv_init(&tq->tq_exit_cv, NULL, CV_DEFAULT, NULL); in taskq_constructor()
790 cv_init(&tq->tq_wait_cv, NULL, CV_DEFAULT, NULL); in taskq_constructor()
791 cv_init(&tq->tq_maxalloc_cv, NULL, CV_DEFAULT, NULL); in taskq_constructor()
793 tq->tq_task.tqent_next = &tq->tq_task; in taskq_constructor()
794 tq->tq_task.tqent_prev = &tq->tq_task; in taskq_constructor()
803 taskq_t *tq = buf; in taskq_destructor() local
805 ASSERT(tq->tq_nthreads == 0); in taskq_destructor()
806 ASSERT(tq->tq_buckets == NULL); in taskq_destructor()
807 ASSERT(tq->tq_tcreates == 0); in taskq_destructor()
808 ASSERT(tq->tq_tdeaths == 0); in taskq_destructor()
810 mutex_destroy(&tq->tq_lock); in taskq_destructor()
811 rw_destroy(&tq->tq_threadlock); in taskq_destructor()
812 cv_destroy(&tq->tq_dispatch_cv); in taskq_destructor()
813 cv_destroy(&tq->tq_exit_cv); in taskq_destructor()
814 cv_destroy(&tq->tq_wait_cv); in taskq_destructor()
815 cv_destroy(&tq->tq_maxalloc_cv); in taskq_destructor()
857 taskq_update_nthreads(taskq_t *tq, uint_t ncpus) in taskq_update_nthreads() argument
859 uint_t newtarget = TASKQ_THREADS_PCT(ncpus, tq->tq_threads_ncpus_pct); in taskq_update_nthreads()
862 ASSERT(MUTEX_HELD(&tq->tq_lock)); in taskq_update_nthreads()
865 ASSERT3U(tq->tq_nthreads_target, !=, 0); in taskq_update_nthreads()
868 ASSERT3U(newtarget, <=, tq->tq_nthreads_max); in taskq_update_nthreads()
869 if (newtarget != tq->tq_nthreads_target) { in taskq_update_nthreads()
870 tq->tq_flags |= TASKQ_CHANGING; in taskq_update_nthreads()
871 tq->tq_nthreads_target = newtarget; in taskq_update_nthreads()
872 cv_broadcast(&tq->tq_dispatch_cv); in taskq_update_nthreads()
873 cv_broadcast(&tq->tq_exit_cv); in taskq_update_nthreads()
879 taskq_cpupct_install(taskq_t *tq, cpupart_t *cpup) in taskq_cpupct_install() argument
881 ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT); in taskq_cpupct_install()
884 mutex_enter(&tq->tq_lock); in taskq_cpupct_install()
885 tq->tq_cpupart = cpup->cp_id; in taskq_cpupct_install()
886 taskq_update_nthreads(tq, cpup->cp_ncpus); in taskq_cpupct_install()
887 mutex_exit(&tq->tq_lock); in taskq_cpupct_install()
889 list_insert_tail(&taskq_cpupct_list, tq); in taskq_cpupct_install()
894 taskq_cpupct_remove(taskq_t *tq) in taskq_cpupct_remove() argument
896 ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT); in taskq_cpupct_remove()
899 list_remove(&taskq_cpupct_list, tq); in taskq_cpupct_remove()
907 taskq_t *tq; in taskq_cpu_setup() local
930 for (tq = list_head(&taskq_cpupct_list); tq != NULL; in taskq_cpu_setup()
931 tq = list_next(&taskq_cpupct_list, tq)) { in taskq_cpu_setup()
933 mutex_enter(&tq->tq_lock); in taskq_cpu_setup()
938 if (tq->tq_cpupart == cp->cp_id) { in taskq_cpu_setup()
939 taskq_update_nthreads(tq, ncpus); in taskq_cpu_setup()
941 mutex_exit(&tq->tq_lock); in taskq_cpu_setup()
979 taskq_ent_alloc(taskq_t *tq, int flags) in taskq_ent_alloc() argument
986 ASSERT(MUTEX_HELD(&tq->tq_lock)); in taskq_ent_alloc()
992 again: if ((tqe = tq->tq_freelist) != NULL && in taskq_ent_alloc()
993 ((flags & TQ_NOALLOC) || tq->tq_nalloc >= tq->tq_minalloc)) { in taskq_ent_alloc()
994 tq->tq_freelist = tqe->tqent_next; in taskq_ent_alloc()
999 if (tq->tq_nalloc >= tq->tq_maxalloc) { in taskq_ent_alloc()
1014 while (tq->tq_freelist == NULL) { in taskq_ent_alloc()
1015 tq->tq_maxalloc_wait++; in taskq_ent_alloc()
1016 wait_rv = cv_timedwait(&tq->tq_maxalloc_cv, in taskq_ent_alloc()
1017 &tq->tq_lock, wait_time); in taskq_ent_alloc()
1018 tq->tq_maxalloc_wait--; in taskq_ent_alloc()
1022 if (tq->tq_freelist) in taskq_ent_alloc()
1026 mutex_exit(&tq->tq_lock); in taskq_ent_alloc()
1030 mutex_enter(&tq->tq_lock); in taskq_ent_alloc()
1032 tq->tq_nalloc++; in taskq_ent_alloc()
1046 taskq_ent_free(taskq_t *tq, taskq_ent_t *tqe) in taskq_ent_free() argument
1048 ASSERT(MUTEX_HELD(&tq->tq_lock)); in taskq_ent_free()
1050 if (tq->tq_nalloc <= tq->tq_minalloc) { in taskq_ent_free()
1051 tqe->tqent_next = tq->tq_freelist; in taskq_ent_free()
1052 tq->tq_freelist = tqe; in taskq_ent_free()
1054 tq->tq_nalloc--; in taskq_ent_free()
1055 mutex_exit(&tq->tq_lock); in taskq_ent_free()
1057 mutex_enter(&tq->tq_lock); in taskq_ent_free()
1060 if (tq->tq_maxalloc_wait) in taskq_ent_free()
1061 cv_signal(&tq->tq_maxalloc_cv); in taskq_ent_free()
1072 taskq_ent_exists(taskq_t *tq, task_func_t func, void *arg) in taskq_ent_exists() argument
1076 ASSERT(MUTEX_HELD(&tq->tq_lock)); in taskq_ent_exists()
1078 for (tqe = tq->tq_task.tqent_next; tqe != &tq->tq_task; in taskq_ent_exists()
1146 taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) in taskq_dispatch() argument
1153 ASSERT(tq != NULL); in taskq_dispatch()
1156 if (!(tq->tq_flags & TASKQ_DYNAMIC)) { in taskq_dispatch()
1164 mutex_enter(&tq->tq_lock); in taskq_dispatch()
1166 TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flags); in taskq_dispatch()
1168 if ((tqe = taskq_ent_alloc(tq, flags)) == NULL) { in taskq_dispatch()
1169 tq->tq_nomem++; in taskq_dispatch()
1170 mutex_exit(&tq->tq_lock); in taskq_dispatch()
1177 TQ_ENQUEUE_FRONT(tq, tqe, func, arg); in taskq_dispatch()
1179 TQ_ENQUEUE(tq, tqe, func, arg); in taskq_dispatch()
1181 mutex_exit(&tq->tq_lock); in taskq_dispatch()
1189 TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flags); in taskq_dispatch()
1191 bsize = tq->tq_nbuckets; in taskq_dispatch()
1198 if ((tqe = taskq_bucket_dispatch(tq->tq_buckets, func, arg)) in taskq_dispatch()
1201 bucket = tq->tq_buckets; in taskq_dispatch()
1214 b = &tq->tq_buckets[h & (bsize - 1)]; in taskq_dispatch()
1215 ASSERT(b->tqbucket_taskq == tq); /* Sanity check */ in taskq_dispatch()
1236 b = &tq->tq_buckets[++h & (bsize - 1)]; in taskq_dispatch()
1237 ASSERT(b->tqbucket_taskq == tq); /* Sanity check */ in taskq_dispatch()
1278 mutex_enter(&tq->tq_lock); in taskq_dispatch()
1279 if (!taskq_ent_exists(tq, taskq_bucket_extend, bucket)) { in taskq_dispatch()
1280 if ((tqe1 = taskq_ent_alloc(tq, TQ_NOSLEEP)) != NULL) { in taskq_dispatch()
1281 TQ_ENQUEUE_FRONT(tq, tqe1, taskq_bucket_extend, bucket); in taskq_dispatch()
1283 tq->tq_nomem++; in taskq_dispatch()
1292 if ((tqe = taskq_ent_alloc(tq, flags)) != NULL) { in taskq_dispatch()
1293 TQ_ENQUEUE(tq, tqe, func, arg); in taskq_dispatch()
1295 tq->tq_nomem++; in taskq_dispatch()
1298 mutex_exit(&tq->tq_lock); in taskq_dispatch()
1304 taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags, in taskq_dispatch_ent() argument
1308 ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC)); in taskq_dispatch_ent()
1318 mutex_enter(&tq->tq_lock); in taskq_dispatch_ent()
1321 TQ_ENQUEUE_FRONT(tq, tqe, func, arg); in taskq_dispatch_ent()
1323 TQ_ENQUEUE(tq, tqe, func, arg); in taskq_dispatch_ent()
1325 mutex_exit(&tq->tq_lock); in taskq_dispatch_ent()
1332 taskq_empty(taskq_t *tq) in taskq_empty() argument
1336 ASSERT3P(tq, !=, curthread->t_taskq); in taskq_empty()
1337 mutex_enter(&tq->tq_lock); in taskq_empty()
1338 rv = (tq->tq_task.tqent_next == &tq->tq_task) && (tq->tq_active == 0); in taskq_empty()
1339 mutex_exit(&tq->tq_lock); in taskq_empty()
1349 taskq_wait(taskq_t *tq) in taskq_wait() argument
1351 ASSERT(tq != curthread->t_taskq); in taskq_wait()
1353 mutex_enter(&tq->tq_lock); in taskq_wait()
1354 while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0) in taskq_wait()
1355 cv_wait(&tq->tq_wait_cv, &tq->tq_lock); in taskq_wait()
1356 mutex_exit(&tq->tq_lock); in taskq_wait()
1358 if (tq->tq_flags & TASKQ_DYNAMIC) { in taskq_wait()
1359 taskq_bucket_t *b = tq->tq_buckets; in taskq_wait()
1361 for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { in taskq_wait()
1371 taskq_wait_id(taskq_t *tq, taskqid_t id __unused) in taskq_wait_id() argument
1373 taskq_wait(tq); in taskq_wait_id()
1384 taskq_suspend(taskq_t *tq) in taskq_suspend() argument
1386 rw_enter(&tq->tq_threadlock, RW_WRITER); in taskq_suspend()
1388 if (tq->tq_flags & TASKQ_DYNAMIC) { in taskq_suspend()
1389 taskq_bucket_t *b = tq->tq_buckets; in taskq_suspend()
1391 for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { in taskq_suspend()
1400 mutex_enter(&tq->tq_lock); in taskq_suspend()
1401 ASSERT(!(tq->tq_flags & TASKQ_SUSPENDED)); in taskq_suspend()
1402 tq->tq_flags |= TASKQ_SUSPENDED; in taskq_suspend()
1403 mutex_exit(&tq->tq_lock); in taskq_suspend()
1410 taskq_suspended(taskq_t *tq) in taskq_suspended() argument
1412 return ((tq->tq_flags & TASKQ_SUSPENDED) != 0); in taskq_suspended()
1419 taskq_resume(taskq_t *tq) in taskq_resume() argument
1421 ASSERT(RW_WRITE_HELD(&tq->tq_threadlock)); in taskq_resume()
1423 if (tq->tq_flags & TASKQ_DYNAMIC) { in taskq_resume()
1424 taskq_bucket_t *b = tq->tq_buckets; in taskq_resume()
1426 for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { in taskq_resume()
1432 mutex_enter(&tq->tq_lock); in taskq_resume()
1433 ASSERT(tq->tq_flags & TASKQ_SUSPENDED); in taskq_resume()
1434 tq->tq_flags &= ~TASKQ_SUSPENDED; in taskq_resume()
1435 mutex_exit(&tq->tq_lock); in taskq_resume()
1437 rw_exit(&tq->tq_threadlock); in taskq_resume()
1441 taskq_member(taskq_t *tq, kthread_t *thread) in taskq_member() argument
1443 return (thread->t_taskq == tq); in taskq_member()
1455 taskq_thread_create(taskq_t *tq) in taskq_thread_create() argument
1458 const boolean_t first = (tq->tq_nthreads == 0); in taskq_thread_create()
1460 ASSERT(MUTEX_HELD(&tq->tq_lock)); in taskq_thread_create()
1461 ASSERT(tq->tq_flags & TASKQ_CHANGING); in taskq_thread_create()
1462 ASSERT(tq->tq_nthreads < tq->tq_nthreads_target); in taskq_thread_create()
1463 ASSERT(!(tq->tq_flags & TASKQ_THREAD_CREATED)); in taskq_thread_create()
1466 tq->tq_flags |= TASKQ_THREAD_CREATED; in taskq_thread_create()
1467 tq->tq_active++; in taskq_thread_create()
1468 mutex_exit(&tq->tq_lock); in taskq_thread_create()
1476 if ((tq->tq_flags & (TASKQ_DUTY_CYCLE | TASKQ_THREADS_LWP)) != 0) { in taskq_thread_create()
1478 ASSERT3P(tq->tq_proc, !=, &p0); in taskq_thread_create()
1479 t = lwp_kernel_create(tq->tq_proc, taskq_thread, tq, TS_RUN, in taskq_thread_create()
1480 tq->tq_pri); in taskq_thread_create()
1482 t = thread_create(NULL, 0, taskq_thread, tq, 0, tq->tq_proc, in taskq_thread_create()
1483 TS_RUN, tq->tq_pri); in taskq_thread_create()
1487 mutex_enter(&tq->tq_lock); in taskq_thread_create()
1496 if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) { in taskq_thread_create()
1497 taskq_cpupct_install(tq, t->t_cpupart); in taskq_thread_create()
1499 mutex_enter(&tq->tq_lock); in taskq_thread_create()
1502 while (tq->tq_nthreads != tq->tq_nthreads_target && in taskq_thread_create()
1503 tq->tq_nthreads < TASKQ_CREATE_ACTIVE_THREADS) { in taskq_thread_create()
1504 cv_wait(&tq->tq_wait_cv, &tq->tq_lock); in taskq_thread_create()
1513 taskq_thread_wait(taskq_t *tq, kmutex_t *mx, kcondvar_t *cv, in taskq_thread_wait() argument
1518 if (!(tq->tq_flags & TASKQ_CPR_SAFE)) { in taskq_thread_wait()
1526 if (!(tq->tq_flags & TASKQ_CPR_SAFE)) { in taskq_thread_wait()
1541 taskq_t *tq = arg; in taskq_thread() local
1547 curthread->t_taskq = tq; /* mark ourselves for taskq_member() */ in taskq_thread()
1549 if (curproc != &p0 && (tq->tq_flags & TASKQ_DUTY_CYCLE)) { in taskq_thread()
1550 sysdc_thread_enter(curthread, tq->tq_DC, in taskq_thread()
1551 (tq->tq_flags & TASKQ_DC_BATCH) ? SYSDC_THREAD_BATCH : 0); in taskq_thread()
1554 if (tq->tq_flags & TASKQ_CPR_SAFE) { in taskq_thread()
1555 CALLB_CPR_INIT_SAFE(curthread, tq->tq_name); in taskq_thread()
1557 CALLB_CPR_INIT(&cprinfo, &tq->tq_lock, callb_generic_cpr, in taskq_thread()
1558 tq->tq_name); in taskq_thread()
1560 mutex_enter(&tq->tq_lock); in taskq_thread()
1561 thread_id = ++tq->tq_nthreads; in taskq_thread()
1562 ASSERT(tq->tq_flags & TASKQ_THREAD_CREATED); in taskq_thread()
1563 ASSERT(tq->tq_flags & TASKQ_CHANGING); in taskq_thread()
1564 tq->tq_flags &= ~TASKQ_THREAD_CREATED; in taskq_thread()
1566 VERIFY3S(thread_id, <=, tq->tq_nthreads_max); in taskq_thread()
1568 if (tq->tq_nthreads_max == 1) in taskq_thread()
1569 tq->tq_thread = curthread; in taskq_thread()
1571 tq->tq_threadlist[thread_id - 1] = curthread; in taskq_thread()
1574 if (tq->tq_nthreads == TASKQ_CREATE_ACTIVE_THREADS) in taskq_thread()
1575 cv_broadcast(&tq->tq_wait_cv); in taskq_thread()
1578 if (tq->tq_flags & TASKQ_CHANGING) { in taskq_thread()
1580 if (thread_id > tq->tq_nthreads_target) { in taskq_thread()
1592 if (thread_id == tq->tq_nthreads || in taskq_thread()
1593 tq->tq_nthreads_target == 0) in taskq_thread()
1597 (void) taskq_thread_wait(tq, &tq->tq_lock, in taskq_thread()
1598 &tq->tq_exit_cv, &cprinfo, -1); in taskq_thread()
1606 if (!(tq->tq_flags & TASKQ_THREAD_CREATED)) { in taskq_thread()
1608 if (tq->tq_nthreads == tq->tq_nthreads_target) { in taskq_thread()
1609 tq->tq_flags &= ~TASKQ_CHANGING; in taskq_thread()
1610 cv_broadcast(&tq->tq_wait_cv); in taskq_thread()
1613 if (tq->tq_nthreads < tq->tq_nthreads_target) { in taskq_thread()
1614 taskq_thread_create(tq); in taskq_thread()
1619 if ((tqe = tq->tq_task.tqent_next) == &tq->tq_task) { in taskq_thread()
1620 if (--tq->tq_active == 0) in taskq_thread()
1621 cv_broadcast(&tq->tq_wait_cv); in taskq_thread()
1622 (void) taskq_thread_wait(tq, &tq->tq_lock, in taskq_thread()
1623 &tq->tq_dispatch_cv, &cprinfo, -1); in taskq_thread()
1624 tq->tq_active++; in taskq_thread()
1630 mutex_exit(&tq->tq_lock); in taskq_thread()
1640 if ((!(tq->tq_flags & TASKQ_DYNAMIC)) && in taskq_thread()
1649 rw_enter(&tq->tq_threadlock, RW_READER); in taskq_thread()
1651 DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq, in taskq_thread()
1654 DTRACE_PROBE2(taskq__exec__end, taskq_t *, tq, in taskq_thread()
1657 rw_exit(&tq->tq_threadlock); in taskq_thread()
1659 mutex_enter(&tq->tq_lock); in taskq_thread()
1660 tq->tq_totaltime += end - start; in taskq_thread()
1661 tq->tq_executed++; in taskq_thread()
1664 taskq_ent_free(tq, tqe); in taskq_thread()
1667 if (tq->tq_nthreads_max == 1) in taskq_thread()
1668 tq->tq_thread = NULL; in taskq_thread()
1670 tq->tq_threadlist[thread_id - 1] = NULL; in taskq_thread()
1673 ASSERT(tq->tq_active > 0); in taskq_thread()
1674 tq->tq_active--; in taskq_thread()
1676 ASSERT(tq->tq_nthreads > 0); in taskq_thread()
1677 tq->tq_nthreads--; in taskq_thread()
1680 cv_broadcast(&tq->tq_exit_cv); in taskq_thread()
1681 if (tq->tq_nthreads == tq->tq_nthreads_target) { in taskq_thread()
1682 if (!(tq->tq_flags & TASKQ_THREAD_CREATED)) in taskq_thread()
1683 tq->tq_flags &= ~TASKQ_CHANGING; in taskq_thread()
1685 cv_broadcast(&tq->tq_wait_cv); in taskq_thread()
1688 ASSERT(!(tq->tq_flags & TASKQ_CPR_SAFE)); in taskq_thread()
1705 taskq_t *tq = bucket->tqbucket_taskq; in taskq_d_thread() local
1711 CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, tq->tq_name); in taskq_d_thread()
1739 DTRACE_PROBE3(taskq__d__exec__start, taskq_t *, tq, in taskq_d_thread()
1742 DTRACE_PROBE3(taskq__d__exec__end, taskq_t *, tq, in taskq_d_thread()
1774 w = taskq_thread_wait(tq, lock, cv, in taskq_d_thread()
1826 mutex_enter(&tq->tq_lock); in taskq_d_thread()
1827 tq->tq_tdeaths++; in taskq_d_thread()
1828 mutex_exit(&tq->tq_lock); in taskq_d_thread()
1908 taskq_t *tq = kmem_cache_alloc(taskq_cache, KM_SLEEP); in taskq_create_common() local
1941 tq->tq_maxsize = nthreads; in taskq_create_common()
1962 tq->tq_threads_ncpus_pct = pct; in taskq_create_common()
1978 (void) strncpy(tq->tq_name, name, TASKQ_NAMELEN + 1); in taskq_create_common()
1979 strident_canon(tq->tq_name, TASKQ_NAMELEN + 1); in taskq_create_common()
1981 tq->tq_flags = flags | TASKQ_CHANGING; in taskq_create_common()
1982 tq->tq_active = 0; in taskq_create_common()
1983 tq->tq_instance = instance; in taskq_create_common()
1984 tq->tq_nthreads_target = nthreads; in taskq_create_common()
1985 tq->tq_nthreads_max = max_nthreads; in taskq_create_common()
1986 tq->tq_minalloc = minalloc; in taskq_create_common()
1987 tq->tq_maxalloc = maxalloc; in taskq_create_common()
1988 tq->tq_nbuckets = bsize; in taskq_create_common()
1989 tq->tq_proc = proc; in taskq_create_common()
1990 tq->tq_pri = pri; in taskq_create_common()
1991 tq->tq_DC = dc; in taskq_create_common()
1992 list_link_init(&tq->tq_cpupct_link); in taskq_create_common()
1995 tq->tq_threadlist = kmem_alloc( in taskq_create_common()
1998 mutex_enter(&tq->tq_lock); in taskq_create_common()
2001 taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP)); in taskq_create_common()
2010 zone_hold(tq->tq_proc->p_zone); in taskq_create_common()
2017 taskq_thread_create(tq); in taskq_create_common()
2018 mutex_exit(&tq->tq_lock); in taskq_create_common()
2025 tq->tq_buckets = bucket; in taskq_create_common()
2032 bucket->tqbucket_taskq = tq; in taskq_create_common()
2050 instance = tq->tq_instance = in taskq_create_common()
2055 if ((tq->tq_kstat = kstat_create("unix", instance, in taskq_create_common()
2056 tq->tq_name, "taskq_d", KSTAT_TYPE_NAMED, in taskq_create_common()
2059 tq->tq_kstat->ks_lock = &taskq_d_kstat_lock; in taskq_create_common()
2060 tq->tq_kstat->ks_data = &taskq_d_kstat; in taskq_create_common()
2061 tq->tq_kstat->ks_update = taskq_d_kstat_update; in taskq_create_common()
2062 tq->tq_kstat->ks_private = tq; in taskq_create_common()
2063 kstat_install(tq->tq_kstat); in taskq_create_common()
2066 if ((tq->tq_kstat = kstat_create("unix", instance, tq->tq_name, in taskq_create_common()
2070 tq->tq_kstat->ks_lock = &taskq_kstat_lock; in taskq_create_common()
2071 tq->tq_kstat->ks_data = &taskq_kstat; in taskq_create_common()
2072 tq->tq_kstat->ks_update = taskq_kstat_update; in taskq_create_common()
2073 tq->tq_kstat->ks_private = tq; in taskq_create_common()
2074 kstat_install(tq->tq_kstat); in taskq_create_common()
2078 return (tq); in taskq_create_common()
2088 taskq_destroy(taskq_t *tq) in taskq_destroy() argument
2090 taskq_bucket_t *b = tq->tq_buckets; in taskq_destroy()
2093 ASSERT(! (tq->tq_flags & TASKQ_CPR_SAFE)); in taskq_destroy()
2098 if (tq->tq_kstat != NULL) { in taskq_destroy()
2099 kstat_delete(tq->tq_kstat); in taskq_destroy()
2100 tq->tq_kstat = NULL; in taskq_destroy()
2106 if (tq->tq_flags & TASKQ_NOINSTANCE) { in taskq_destroy()
2107 vmem_free(taskq_id_arena, (void *)(uintptr_t)(tq->tq_instance), in taskq_destroy()
2109 tq->tq_instance = 0; in taskq_destroy()
2115 if (tq->tq_flags & TASKQ_THREADS_CPU_PCT) { in taskq_destroy()
2116 taskq_cpupct_remove(tq); in taskq_destroy()
2122 taskq_wait(tq); in taskq_destroy()
2124 mutex_enter(&tq->tq_lock); in taskq_destroy()
2125 ASSERT((tq->tq_task.tqent_next == &tq->tq_task) && in taskq_destroy()
2126 (tq->tq_active == 0)); in taskq_destroy()
2129 tq->tq_nthreads_target = 0; in taskq_destroy()
2131 tq->tq_flags |= TASKQ_CHANGING; in taskq_destroy()
2132 cv_broadcast(&tq->tq_dispatch_cv); in taskq_destroy()
2133 cv_broadcast(&tq->tq_exit_cv); in taskq_destroy()
2135 while (tq->tq_nthreads != 0) in taskq_destroy()
2136 cv_wait(&tq->tq_wait_cv, &tq->tq_lock); in taskq_destroy()
2138 if (tq->tq_nthreads_max != 1) in taskq_destroy()
2139 kmem_free(tq->tq_threadlist, sizeof (kthread_t *) * in taskq_destroy()
2140 tq->tq_nthreads_max); in taskq_destroy()
2142 tq->tq_minalloc = 0; in taskq_destroy()
2143 while (tq->tq_nalloc != 0) in taskq_destroy()
2144 taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP)); in taskq_destroy()
2146 mutex_exit(&tq->tq_lock); in taskq_destroy()
2151 for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { in taskq_destroy()
2177 if (tq->tq_buckets != NULL) { in taskq_destroy()
2178 ASSERT(tq->tq_flags & TASKQ_DYNAMIC); in taskq_destroy()
2179 kmem_free(tq->tq_buckets, in taskq_destroy()
2180 sizeof (taskq_bucket_t) * tq->tq_nbuckets); in taskq_destroy()
2183 tq->tq_buckets = NULL; in taskq_destroy()
2184 tq->tq_tcreates = 0; in taskq_destroy()
2185 tq->tq_tdeaths = 0; in taskq_destroy()
2187 ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC)); in taskq_destroy()
2194 zone_rele(tq->tq_proc->p_zone); in taskq_destroy()
2196 tq->tq_threads_ncpus_pct = 0; in taskq_destroy()
2197 tq->tq_totaltime = 0; in taskq_destroy()
2198 tq->tq_tasks = 0; in taskq_destroy()
2199 tq->tq_maxtasks = 0; in taskq_destroy()
2200 tq->tq_executed = 0; in taskq_destroy()
2201 kmem_cache_free(taskq_cache, tq); in taskq_destroy()
2218 taskq_t *tq = b->tqbucket_taskq; in taskq_bucket_extend() local
2222 mutex_enter(&tq->tq_lock); in taskq_bucket_extend()
2225 tq->tq_nomem++; in taskq_bucket_extend()
2226 mutex_exit(&tq->tq_lock); in taskq_bucket_extend()
2233 if (tq->tq_tcreates++ - tq->tq_tdeaths > tq->tq_maxsize) { in taskq_bucket_extend()
2234 tq->tq_tcreates--; in taskq_bucket_extend()
2235 mutex_exit(&tq->tq_lock); in taskq_bucket_extend()
2238 mutex_exit(&tq->tq_lock); in taskq_bucket_extend()
2243 mutex_enter(&tq->tq_lock); in taskq_bucket_extend()
2244 tq->tq_nomem++; in taskq_bucket_extend()
2245 tq->tq_tcreates--; in taskq_bucket_extend()
2246 mutex_exit(&tq->tq_lock); in taskq_bucket_extend()
2258 if ((tq->tq_flags & TASKQ_THREADS_LWP) != 0) { in taskq_bucket_extend()
2260 ASSERT3P(tq->tq_proc, !=, &p0); in taskq_bucket_extend()
2261 t = lwp_kernel_create(tq->tq_proc, taskq_d_thread, in taskq_bucket_extend()
2262 tqe, TS_STOPPED, tq->tq_pri); in taskq_bucket_extend()
2265 0, tq->tq_proc, TS_STOPPED, tq->tq_pri); in taskq_bucket_extend()
2268 t->t_taskq = tq; /* mark thread as a taskq_member() */ in taskq_bucket_extend()
2292 proc_t *p = tq->tq_proc; in taskq_bucket_extend()
2309 taskq_t *tq = ksp->ks_private; in taskq_kstat_update() local
2314 tqsp->tq_pid.value.ui64 = tq->tq_proc->p_pid; in taskq_kstat_update()
2315 tqsp->tq_tasks.value.ui64 = tq->tq_tasks; in taskq_kstat_update()
2316 tqsp->tq_executed.value.ui64 = tq->tq_executed; in taskq_kstat_update()
2317 tqsp->tq_maxtasks.value.ui64 = tq->tq_maxtasks; in taskq_kstat_update()
2318 tqsp->tq_totaltime.value.ui64 = tq->tq_totaltime; in taskq_kstat_update()
2319 tqsp->tq_nactive.value.ui64 = tq->tq_active; in taskq_kstat_update()
2320 tqsp->tq_nalloc.value.ui64 = tq->tq_nalloc; in taskq_kstat_update()
2321 tqsp->tq_pri.value.ui64 = tq->tq_pri; in taskq_kstat_update()
2322 tqsp->tq_nthreads.value.ui64 = tq->tq_nthreads; in taskq_kstat_update()
2323 tqsp->tq_nomem.value.ui64 = tq->tq_nomem; in taskq_kstat_update()
2331 taskq_t *tq = ksp->ks_private; in taskq_d_kstat_update() local
2332 taskq_bucket_t *b = tq->tq_buckets; in taskq_d_kstat_update()
2338 ASSERT(tq->tq_flags & TASKQ_DYNAMIC); in taskq_d_kstat_update()
2340 tqsp->tqd_btasks.value.ui64 = tq->tq_tasks; in taskq_d_kstat_update()
2341 tqsp->tqd_bexecuted.value.ui64 = tq->tq_executed; in taskq_d_kstat_update()
2342 tqsp->tqd_bmaxtasks.value.ui64 = tq->tq_maxtasks; in taskq_d_kstat_update()
2343 tqsp->tqd_bnalloc.value.ui64 = tq->tq_nalloc; in taskq_d_kstat_update()
2344 tqsp->tqd_bnactive.value.ui64 = tq->tq_active; in taskq_d_kstat_update()
2345 tqsp->tqd_btotaltime.value.ui64 = tq->tq_totaltime; in taskq_d_kstat_update()
2346 tqsp->tqd_pri.value.ui64 = tq->tq_pri; in taskq_d_kstat_update()
2347 tqsp->tqd_nomem.value.ui64 = tq->tq_nomem; in taskq_d_kstat_update()
2361 for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { in taskq_d_kstat_update()