718 #define	TQ_APPEND(l, tqe) {					\  argument
719 	tqe->tqent_next = &l;					\
720 	tqe->tqent_prev = l.tqent_prev;				\
721 	tqe->tqent_next->tqent_prev = tqe;			\
722 	tqe->tqent_prev->tqent_next = tqe;			\
727 #define	TQ_PREPEND(l, tqe) {					\  argument
728 	tqe->tqent_next = l.tqent_next;				\
729 	tqe->tqent_prev = &l;					\
730 	tqe->tqent_next->tqent_prev = tqe;			\
731 	tqe->tqent_prev->tqent_next = tqe;			\
737 #define	TQ_DO_ENQUEUE(tq, tqe, func, arg, front) {			\  argument
741 		TQ_PREPEND(tq->tq_task, tqe);				\
743 		TQ_APPEND(tq->tq_task, tqe);				\
745 	tqe->tqent_func = (func);					\
746 	tqe->tqent_arg = (arg);						\
751 	DTRACE_PROBE2(taskq__enqueue, taskq_t *, tq, taskq_ent_t *, tqe); \
754 #define	TQ_ENQUEUE(tq, tqe, func, arg)					\  argument
755 	TQ_DO_ENQUEUE(tq, tqe, func, arg, 0)
757 #define	TQ_ENQUEUE_FRONT(tq, tqe, func, arg)				\  argument
758 	TQ_DO_ENQUEUE(tq, tqe, func, arg, 1)
813 	taskq_ent_t *tqe = buf;  in taskq_ent_constructor()  local
815 	tqe->tqent_thread = NULL;  in taskq_ent_constructor()
816 	cv_init(&tqe->tqent_cv, NULL, CV_DEFAULT, NULL);  in taskq_ent_constructor()
825 	taskq_ent_t *tqe = buf;  in taskq_ent_destructor()  local
827 	ASSERT(tqe->tqent_thread == NULL);  in taskq_ent_destructor()
828 	cv_destroy(&tqe->tqent_cv);  in taskq_ent_destructor()
973 	taskq_ent_t *tqe;  in taskq_ent_alloc()  local
983 again:	if ((tqe = tq->tq_freelist) != NULL &&  in taskq_ent_alloc()
985 		tq->tq_freelist = tqe->tqent_next;  in taskq_ent_alloc()
1019 		tqe = kmem_cache_alloc(taskq_ent_cache, kmflags);  in taskq_ent_alloc()
1022 		if (tqe != NULL)  in taskq_ent_alloc()
1025 	return (tqe);  in taskq_ent_alloc()
1037 taskq_ent_free(taskq_t *tq, taskq_ent_t *tqe)  in taskq_ent_free()  argument
1042 		tqe->tqent_next = tq->tq_freelist;  in taskq_ent_free()
1043 		tq->tq_freelist = tqe;  in taskq_ent_free()
1047 		kmem_cache_free(taskq_ent_cache, tqe);  in taskq_ent_free()
1065 	taskq_ent_t	*tqe;  in taskq_ent_exists()  local
1069 	for (tqe = tq->tq_task.tqent_next; tqe != &tq->tq_task;  in taskq_ent_exists()
1070 	    tqe = tqe->tqent_next)  in taskq_ent_exists()
1071 		if ((tqe->tqent_func == func) && (tqe->tqent_arg == arg))  in taskq_ent_exists()
1087 	taskq_ent_t *tqe;  in taskq_bucket_dispatch()  local
1103 		tqe = b->tqbucket_freelist.tqent_prev;  in taskq_bucket_dispatch()
1105 		ASSERT(tqe != &b->tqbucket_freelist);  in taskq_bucket_dispatch()
1106 		ASSERT(tqe->tqent_thread != NULL);  in taskq_bucket_dispatch()
1108 		tqe->tqent_prev->tqent_next = tqe->tqent_next;  in taskq_bucket_dispatch()
1109 		tqe->tqent_next->tqent_prev = tqe->tqent_prev;  in taskq_bucket_dispatch()
1112 		tqe->tqent_func = func;  in taskq_bucket_dispatch()
1113 		tqe->tqent_arg = arg;  in taskq_bucket_dispatch()
1115 		cv_signal(&tqe->tqent_cv);  in taskq_bucket_dispatch()
1117 		    taskq_ent_t *, tqe);  in taskq_bucket_dispatch()
1119 		tqe = NULL;  in taskq_bucket_dispatch()
1123 	return (tqe);  in taskq_bucket_dispatch()
1140 	taskq_ent_t *tqe = NULL;  in taskq_dispatch()  local
1159 		if ((tqe = taskq_ent_alloc(tq, flags)) == NULL) {  in taskq_dispatch()
1164 		tqe->tqent_un.tqent_flags = 0;  in taskq_dispatch()
1167 			TQ_ENQUEUE_FRONT(tq, tqe, func, arg);  in taskq_dispatch()
1169 			TQ_ENQUEUE(tq, tqe, func, arg);  in taskq_dispatch()
1172 		return ((taskqid_t)tqe);  in taskq_dispatch()
1188 		if ((tqe = taskq_bucket_dispatch(tq->tq_buckets, func, arg))  in taskq_dispatch()
1190 			return ((taskqid_t)tqe);	/* Fastpath */  in taskq_dispatch()
1213 			if ((tqe = taskq_bucket_dispatch(b, func, arg)) != NULL)  in taskq_dispatch()
1214 				return ((taskqid_t)tqe);	/* Fastpath */  in taskq_dispatch()
1231 				tqe = taskq_bucket_dispatch(b, func, arg);  in taskq_dispatch()
1235 		} while ((tqe == NULL) && (loopcount > 0));  in taskq_dispatch()
1246 	if ((tqe == NULL) && !(flags & TQ_NOSLEEP)) {  in taskq_dispatch()
1257 		if ((tqe = taskq_bucket_dispatch(bucket, func, arg)) != NULL)  in taskq_dispatch()
1258 			return ((taskqid_t)tqe);  in taskq_dispatch()
1281 	if ((tqe == NULL) && !(flags & TQ_NOQUEUE)) {  in taskq_dispatch()
1282 		if ((tqe = taskq_ent_alloc(tq, flags)) != NULL) {  in taskq_dispatch()
1283 			TQ_ENQUEUE(tq, tqe, func, arg);  in taskq_dispatch()
1290 	return ((taskqid_t)tqe);  in taskq_dispatch()
1295     taskq_ent_t *tqe)  in taskq_dispatch_ent()  argument
1304 	tqe->tqent_un.tqent_flags |= TQENT_FLAG_PREALLOC;  in taskq_dispatch_ent()
1311 		TQ_ENQUEUE_FRONT(tq, tqe, func, arg);  in taskq_dispatch_ent()
1313 		TQ_ENQUEUE(tq, tqe, func, arg);  in taskq_dispatch_ent()
1509 	taskq_ent_t *tqe;  in taskq_thread()  local
1586 		if ((tqe = tq->tq_task.tqent_next) == &tq->tq_task) {  in taskq_thread()
1595 		tqe->tqent_prev->tqent_next = tqe->tqent_next;  in taskq_thread()
1596 		tqe->tqent_next->tqent_prev = tqe->tqent_prev;  in taskq_thread()
1608 		    (tqe->tqent_un.tqent_flags & TQENT_FLAG_PREALLOC)) {  in taskq_thread()
1610 			tqe->tqent_next = tqe->tqent_prev = NULL;  in taskq_thread()
1619 		    taskq_ent_t *, tqe);  in taskq_thread()
1620 		tqe->tqent_func(tqe->tqent_arg);  in taskq_thread()
1622 		    taskq_ent_t *, tqe);  in taskq_thread()
1631 			taskq_ent_free(tq, tqe);  in taskq_thread()
1669 taskq_d_thread(taskq_ent_t *tqe)  in taskq_d_thread()  argument
1671 	taskq_bucket_t	*bucket = tqe->tqent_un.tqent_bucket;  in taskq_d_thread()
1674 	kcondvar_t	*cv = &tqe->tqent_cv;  in taskq_d_thread()
1687 		if (tqe->tqent_func != NULL) {  in taskq_d_thread()
1707 			    taskq_bucket_t *, bucket, taskq_ent_t *, tqe);  in taskq_d_thread()
1708 			tqe->tqent_func(tqe->tqent_arg);  in taskq_d_thread()
1710 			    taskq_bucket_t *, bucket, taskq_ent_t *, tqe);  in taskq_d_thread()
1718 			tqe->tqent_func = NULL;  in taskq_d_thread()
1719 			TQ_APPEND(bucket->tqbucket_freelist, tqe);  in taskq_d_thread()
1756 		ASSERT((tqe->tqent_func != NULL) ||  in taskq_d_thread()
1759 		ASSERT((tqe->tqent_func == NULL) ||  in taskq_d_thread()
1768 		if ((tqe->tqent_func == NULL) &&  in taskq_d_thread()
1779 			tqe->tqent_prev->tqent_next = tqe->tqent_next;  in taskq_d_thread()
1780 			tqe->tqent_next->tqent_prev = tqe->tqent_prev;  in taskq_d_thread()
1786 			tqe->tqent_thread = NULL;  in taskq_d_thread()
1791 			kmem_cache_free(taskq_ent_cache, tqe);  in taskq_d_thread()
2105 		taskq_ent_t *tqe;  in taskq_destroy()  local
2112 		for (tqe = b->tqbucket_freelist.tqent_next;  in taskq_destroy()
2113 		    tqe != &b->tqbucket_freelist; tqe = tqe->tqent_next)  in taskq_destroy()
2114 			cv_signal(&tqe->tqent_cv);  in taskq_destroy()
2169 	taskq_ent_t *tqe;  in taskq_bucket_extend()  local
2191 	tqe = kmem_cache_alloc(taskq_ent_cache, KM_NOSLEEP);  in taskq_bucket_extend()
2193 	if (tqe == NULL) {  in taskq_bucket_extend()
2201 	ASSERT(tqe->tqent_thread == NULL);  in taskq_bucket_extend()
2203 	tqe->tqent_un.tqent_bucket = b;  in taskq_bucket_extend()
2209 	tqe->tqent_thread = thread_create(NULL, 0, taskq_d_thread, tqe,  in taskq_bucket_extend()
2216 	tqe->tqent_func = NULL;  in taskq_bucket_extend()
2217 	TQ_APPEND(b->tqbucket_freelist, tqe);  in taskq_bucket_extend()
2232 	thread_lock(tqe->tqent_thread);  in taskq_bucket_extend()
2233 	tqe->tqent_thread->t_taskq = tq;  in taskq_bucket_extend()
2234 	tqe->tqent_thread->t_schedflag |= TS_ALLSTART;  in taskq_bucket_extend()
2235 	setrun_locked(tqe->tqent_thread);  in taskq_bucket_extend()
2236 	thread_unlock(tqe->tqent_thread);  in taskq_bucket_extend()