1333 taskq_backlog_dispatch(taskq_bucket_t *bucket, task_func_t func, void *arg,  in taskq_backlog_dispatch()  argument
1346 	mutex_enter(&bucket->tqbucket_lock);  in taskq_backlog_dispatch()
1347 	taskq_backlog_enqueue(bucket, tqe, flags);  in taskq_backlog_dispatch()
1348 	mutex_exit(&bucket->tqbucket_lock);  in taskq_backlog_dispatch()
1354 taskq_backlog_enqueue(taskq_bucket_t *bucket, taskq_ent_t *tqe, int flags)  in taskq_backlog_enqueue()  argument
1357 	ASSERT(MUTEX_HELD(&bucket->tqbucket_lock));  in taskq_backlog_enqueue()
1359 	tqe->tqent_un.tqent_bucket = bucket;  in taskq_backlog_enqueue()
1361 		TQ_PREPEND(bucket->tqbucket_backlog, tqe);  in taskq_backlog_enqueue()
1363 		TQ_APPEND(bucket->tqbucket_backlog, tqe);  in taskq_backlog_enqueue()
1365 	bucket->tqbucket_nbacklog++;  in taskq_backlog_enqueue()
1369 	    taskq_bucket_t *, bucket,  in taskq_backlog_enqueue()
1371 	TQ_STAT(bucket, tqs_overflow);  in taskq_backlog_enqueue()
1373 	if (bucket->tqbucket_stat.tqs_maxbacklog <  in taskq_backlog_enqueue()
1374 	    bucket->tqbucket_nbacklog) {  in taskq_backlog_enqueue()
1375 		bucket->tqbucket_stat.tqs_maxbacklog =  in taskq_backlog_enqueue()
1376 		    bucket->tqbucket_nbacklog;  in taskq_backlog_enqueue()
1388 	if (bucket->tqbucket_nfree != 0) {  in taskq_backlog_enqueue()
1390 		itqe = bucket->tqbucket_freelist.tqent_prev;  in taskq_backlog_enqueue()
1408 	taskq_bucket_t *bucket = NULL;	/* Which bucket needs extension */  in taskq_dispatch()  local
1463 		bucket = tq->tq_buckets;  in taskq_dispatch()
1467 		bucket = &tq->tq_buckets[h & (bsize - 1)];  in taskq_dispatch()
1468 		ASSERT(bucket->tqbucket_taskq == tq);	/* Sanity check */  in taskq_dispatch()
1475 		if (bucket->tqbucket_nfree != 0) {  in taskq_dispatch()
1476 			tqe = taskq_bucket_dispatch(bucket, func, arg);  in taskq_dispatch()
1480 			TQ_STAT(bucket, tqs_misses);  in taskq_dispatch()
1488 	tqe = taskq_idlebucket_dispatch(bucket, func, arg);  in taskq_dispatch()
1510 		while (taskq_bucket_extend(bucket) != NULL) {  in taskq_dispatch()
1511 			TQ_STAT(bucket, tqs_disptcreates);  in taskq_dispatch()
1512 			tqe = taskq_bucket_dispatch(bucket, func, arg);  in taskq_dispatch()
1527 		tqe = taskq_backlog_dispatch(bucket, func, arg, flags);  in taskq_dispatch()
1539 	if (!taskq_ent_exists(tq, taskq_bucket_overflow, bucket)) {  in taskq_dispatch()
1542 			TQ_ENQUEUE(tq, tqe1, taskq_bucket_overflow, bucket);  in taskq_dispatch()
1984     taskq_bucket_t *bucket, taskq_t *tq)  in taskq_d_svc_bucket()  argument
1986 	kmutex_t	*lock = &bucket->tqbucket_lock;  in taskq_d_svc_bucket()
2002 		ASSERT(bucket->tqbucket_nfree > 0);  in taskq_d_svc_bucket()
2008 		if (bucket->tqbucket_nbacklog > 0) {  in taskq_d_svc_bucket()
2010 			bucket->tqbucket_nfree--;  in taskq_d_svc_bucket()
2012 			bucket->tqbucket_nalloc++;  in taskq_d_svc_bucket()
2027 		bucket->tqbucket_flags &= ~TQBUCKET_REDIRECT;  in taskq_d_svc_bucket()
2039 		bucket->tqbucket_nalloc++;  in taskq_d_svc_bucket()
2053 			ASSERT(bucket->tqbucket_nalloc > 0);  in taskq_d_svc_bucket()
2060 				bucket->tqbucket_nalloc--;  in taskq_d_svc_bucket()
2070 			    taskq_bucket_t *, bucket, taskq_ent_t *, tqe);  in taskq_d_svc_bucket()
2073 			    taskq_bucket_t *, bucket, taskq_ent_t *, tqe);  in taskq_d_svc_bucket()
2076 			bucket->tqbucket_totaltime += end - start;  in taskq_d_svc_bucket()
2085 		if (bucket->tqbucket_nbacklog > 0) {  in taskq_d_svc_bucket()
2093 			ASSERT(bucket->tqbucket_nalloc > 0);  in taskq_d_svc_bucket()
2095 			ASSERT(!IS_EMPTY(bucket->tqbucket_backlog));  in taskq_d_svc_bucket()
2096 			bltqe = bucket->tqbucket_backlog.tqent_next;  in taskq_d_svc_bucket()
2098 			bucket->tqbucket_nbacklog--;  in taskq_d_svc_bucket()
2101 			    taskq_bucket_t *, bucket,  in taskq_d_svc_bucket()
2124 		ASSERT(bucket->tqbucket_nalloc > 0);  in taskq_d_svc_bucket()
2125 		bucket->tqbucket_nalloc--;  in taskq_d_svc_bucket()
2127 		TQ_APPEND(bucket->tqbucket_freelist, tqe);  in taskq_d_svc_bucket()
2128 		bucket->tqbucket_nfree++;  in taskq_d_svc_bucket()
2134 		cv_signal(&bucket->tqbucket_cv);  in taskq_d_svc_bucket()
2148 		if ((bucket->tqbucket_flags & TQBUCKET_CLOSE) != 0)  in taskq_d_svc_bucket()
2177 			ASSERT(bucket->tqbucket_nfree > 0);  in taskq_d_svc_bucket()
2189 			if (bucket->tqbucket_nbacklog > 0) {  in taskq_d_svc_bucket()
2191 				bucket->tqbucket_nfree--;  in taskq_d_svc_bucket()
2193 				bucket->tqbucket_nalloc++;  in taskq_d_svc_bucket()
2224 	ASSERT(bucket->tqbucket_nfree > 0);  in taskq_d_svc_bucket()
2225 	bucket->tqbucket_nfree--;  in taskq_d_svc_bucket()
2226 	cv_signal(&bucket->tqbucket_cv);  in taskq_d_svc_bucket()
2240 	taskq_bucket_t	*bucket = tqe->tqent_un.tqent_bucket;  in taskq_d_thread()  local
2241 	taskq_t		*tq = bucket->tqbucket_taskq;  in taskq_d_thread()
2254 	bucket = tqe->tqent_un.tqent_bucket;  in taskq_d_thread()
2261 	if (bucket == idle_bucket) {  in taskq_d_thread()
2279 		bucket = tqe->tqent_un.tqent_bucket;  in taskq_d_thread()
2280 		VERIFY3P(bucket, >=, tq->tq_buckets);  in taskq_d_thread()
2281 		VERIFY3P(bucket, <, idle_bucket);  in taskq_d_thread()
2284 		taskq_d_svc_bucket(tqe, bucket, tq);  in taskq_d_thread()
2321 		VERIFY3P(bucket, >=, tq->tq_buckets);  in taskq_d_thread()
2322 		VERIFY3P(bucket, <, idle_bucket);  in taskq_d_thread()
2324 		b = bucket;  in taskq_d_thread()
2340 		} while (b != bucket);  in taskq_d_thread()
2426 			bucket = tq->tq_buckets;  in taskq_d_thread()
2430 			tqe->tqent_un.tqent_bucket = bucket;  in taskq_d_thread()
2690 		taskq_bucket_t *bucket = kmem_zalloc(sizeof (taskq_bucket_t) *  in taskq_create_common()  local
2692 		taskq_bucket_t *idle_bucket = &bucket[bsize];  in taskq_create_common()
2695 		tq->tq_buckets = bucket;  in taskq_create_common()
2698 		for (b_id = 0; b_id < (bsize + 1); b_id++, bucket++) {  in taskq_create_common()
2699 			mutex_init(&bucket->tqbucket_lock, NULL, MUTEX_DEFAULT,  in taskq_create_common()
2701 			cv_init(&bucket->tqbucket_cv, NULL, CV_DEFAULT, NULL);  in taskq_create_common()
2702 			bucket->tqbucket_taskq = tq;  in taskq_create_common()
2703 			TQ_LIST_INIT(bucket->tqbucket_freelist);  in taskq_create_common()
2704 			TQ_LIST_INIT(bucket->tqbucket_backlog);  in taskq_create_common()
3073 taskq_bucket_redist(taskq_bucket_t *bucket)  in taskq_bucket_redist()  argument
3075 	taskq_t *tq = bucket->tqbucket_taskq;  in taskq_bucket_redist()
3081 	VERIFY3P(bucket, >=, tq->tq_buckets);  in taskq_bucket_redist()
3082 	VERIFY3P(bucket, <, idle_bucket);  in taskq_bucket_redist()
3095 	mutex_enter(&bucket->tqbucket_lock);  in taskq_bucket_redist()
3096 	nthreads = bucket->tqbucket_nalloc + bucket->tqbucket_nfree;  in taskq_bucket_redist()
3097 	if (nthreads > 0 || bucket->tqbucket_nbacklog == 0 ||  in taskq_bucket_redist()
3098 	    (bucket->tqbucket_flags & TQBUCKET_REDIRECT) != 0) {  in taskq_bucket_redist()
3099 		mutex_exit(&bucket->tqbucket_lock);  in taskq_bucket_redist()
3103 	bucket->tqbucket_flags |= TQBUCKET_REDIRECT;  in taskq_bucket_redist()
3104 	mutex_exit(&bucket->tqbucket_lock);  in taskq_bucket_redist()
3112 	tqe->tqent_arg = bucket; /* redirected to */  in taskq_bucket_redist()
3120 	db = bucket;  in taskq_bucket_redist()
3125 		if (db == bucket)  in taskq_bucket_redist()
3142 	    taskq_bucket_t *, bucket);  in taskq_bucket_redist()
3145 	mutex_enter(&bucket->tqbucket_lock);  in taskq_bucket_redist()
3146 	bucket->tqbucket_flags &= ~TQBUCKET_REDIRECT;  in taskq_bucket_redist()
3147 	mutex_exit(&bucket->tqbucket_lock);  in taskq_bucket_redist()
3158 		itqe = bucket->tqbucket_freelist.tqent_prev;  in taskq_bucket_redist()
3164 	    taskq_bucket_t *, bucket, taskq_ent_t *, tqe);  in taskq_bucket_redist()