51 static int	task_is_running(struct gtaskqueue *queue, struct gtask *gtask);
52 static void	gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask);
127 	struct gtaskqueue *queue;  in _gtaskqueue_create()  local
136 	queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags | M_ZERO);  in _gtaskqueue_create()
137 	if (!queue) {  in _gtaskqueue_create()
142 	STAILQ_INIT(&queue->tq_queue);  in _gtaskqueue_create()
143 	LIST_INIT(&queue->tq_active);  in _gtaskqueue_create()
144 	queue->tq_enqueue = enqueue;  in _gtaskqueue_create()
145 	queue->tq_context = context;  in _gtaskqueue_create()
146 	queue->tq_name = tq_name;  in _gtaskqueue_create()
147 	queue->tq_spin = (mtxflags & MTX_SPIN) != 0;  in _gtaskqueue_create()
148 	queue->tq_flags |= TQ_FLAGS_ACTIVE;  in _gtaskqueue_create()
150 		queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE;  in _gtaskqueue_create()
151 	mtx_init(&queue->tq_mutex, tq_name, NULL, mtxflags);  in _gtaskqueue_create()
153 	return (queue);  in _gtaskqueue_create()
170 gtaskqueue_free(struct gtaskqueue *queue)  in gtaskqueue_free()  argument
173 	TQ_LOCK(queue);  in gtaskqueue_free()
174 	queue->tq_flags &= ~TQ_FLAGS_ACTIVE;  in gtaskqueue_free()
175 	gtaskqueue_terminate(queue->tq_threads, queue);  in gtaskqueue_free()
176 	KASSERT(LIST_EMPTY(&queue->tq_active), ("Tasks still running?"));  in gtaskqueue_free()
177 	KASSERT(queue->tq_callouts == 0, ("Armed timeout tasks"));  in gtaskqueue_free()
178 	mtx_destroy(&queue->tq_mutex);  in gtaskqueue_free()
179 	free(queue->tq_threads, M_GTASKQUEUE);  in gtaskqueue_free()
180 	free(queue->tq_name, M_GTASKQUEUE);  in gtaskqueue_free()
181 	free(queue, M_GTASKQUEUE);  in gtaskqueue_free()
190 	struct gtaskqueue *queue = grouptask->gt_taskqueue;  in grouptask_block()  local
194 	if (queue == NULL) {  in grouptask_block()
196 		panic("queue == NULL");  in grouptask_block()
199 	TQ_LOCK(queue);  in grouptask_block()
201   	gtaskqueue_drain_locked(queue, gtask);  in grouptask_block()
202 	TQ_UNLOCK(queue);  in grouptask_block()
208 	struct gtaskqueue *queue = grouptask->gt_taskqueue;  in grouptask_unblock()  local
212 	if (queue == NULL) {  in grouptask_unblock()
214 		panic("queue == NULL");  in grouptask_unblock()
217 	TQ_LOCK(queue);  in grouptask_unblock()
219 	TQ_UNLOCK(queue);  in grouptask_unblock()
223 grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask)  in grouptaskqueue_enqueue()  argument
226 	if (queue == NULL) {  in grouptaskqueue_enqueue()
228 		panic("queue == NULL");  in grouptaskqueue_enqueue()
231 	TQ_LOCK(queue);  in grouptaskqueue_enqueue()
233 		TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
237 		TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
240 	STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link);  in grouptaskqueue_enqueue()
242 	TQ_UNLOCK(queue);  in grouptaskqueue_enqueue()
243 	if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0)  in grouptaskqueue_enqueue()
244 		queue->tq_enqueue(queue->tq_context);  in grouptaskqueue_enqueue()
259 gtaskqueue_drain_tq_queue(struct gtaskqueue *queue)  in gtaskqueue_drain_tq_queue()  argument
263 	if (STAILQ_EMPTY(&queue->tq_queue))  in gtaskqueue_drain_tq_queue()
272 	 * queue lock.  in gtaskqueue_drain_tq_queue()
275 	STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link);  in gtaskqueue_drain_tq_queue()
283 		TQ_SLEEP(queue, &t_barrier, "gtq_qdrain");  in gtaskqueue_drain_tq_queue()
292 gtaskqueue_drain_tq_active(struct gtaskqueue *queue)  in gtaskqueue_drain_tq_active()  argument
297 	if (LIST_EMPTY(&queue->tq_active))  in gtaskqueue_drain_tq_active()
301 	queue->tq_callouts++;  in gtaskqueue_drain_tq_active()
304 	seq = queue->tq_seq;  in gtaskqueue_drain_tq_active()
306 	LIST_FOREACH(tb, &queue->tq_active, tb_link) {  in gtaskqueue_drain_tq_active()
308 			TQ_SLEEP(queue, tb->tb_running, "gtq_adrain");  in gtaskqueue_drain_tq_active()
314 	queue->tq_callouts--;  in gtaskqueue_drain_tq_active()
315 	if ((queue->tq_flags & TQ_FLAGS_ACTIVE) == 0)  in gtaskqueue_drain_tq_active()
316 		wakeup_one(queue->tq_threads);  in gtaskqueue_drain_tq_active()
320 gtaskqueue_block(struct gtaskqueue *queue)  in gtaskqueue_block()  argument
323 	TQ_LOCK(queue);  in gtaskqueue_block()
324 	queue->tq_flags |= TQ_FLAGS_BLOCKED;  in gtaskqueue_block()
325 	TQ_UNLOCK(queue);  in gtaskqueue_block()
329 gtaskqueue_unblock(struct gtaskqueue *queue)  in gtaskqueue_unblock()  argument
332 	TQ_LOCK(queue);  in gtaskqueue_unblock()
333 	queue->tq_flags &= ~TQ_FLAGS_BLOCKED;  in gtaskqueue_unblock()
334 	if (!STAILQ_EMPTY(&queue->tq_queue))  in gtaskqueue_unblock()
335 		queue->tq_enqueue(queue->tq_context);  in gtaskqueue_unblock()
336 	TQ_UNLOCK(queue);  in gtaskqueue_unblock()
340 gtaskqueue_run_locked(struct gtaskqueue *queue)  in gtaskqueue_run_locked()  argument
347 	KASSERT(queue != NULL, ("tq is NULL"));  in gtaskqueue_run_locked()
348 	TQ_ASSERT_LOCKED(queue);  in gtaskqueue_run_locked()
350 	LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);  in gtaskqueue_run_locked()
353 	while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) {  in gtaskqueue_run_locked()
354 		STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);  in gtaskqueue_run_locked()
357 		tb.tb_seq = ++queue->tq_seq;  in gtaskqueue_run_locked()
358 		TQ_UNLOCK(queue);  in gtaskqueue_run_locked()
370 		TQ_LOCK(queue);  in gtaskqueue_run_locked()
379 task_is_running(struct gtaskqueue *queue, struct gtask *gtask)  in task_is_running()  argument
383 	TQ_ASSERT_LOCKED(queue);  in task_is_running()
384 	LIST_FOREACH(tb, &queue->tq_active, tb_link) {  in task_is_running()
392 gtaskqueue_cancel_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel_locked()  argument
396 		STAILQ_REMOVE(&queue->tq_queue, gtask, gtask, ta_link);  in gtaskqueue_cancel_locked()
398 	return (task_is_running(queue, gtask) ? EBUSY : 0);  in gtaskqueue_cancel_locked()
402 gtaskqueue_cancel(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel()  argument
406 	TQ_LOCK(queue);  in gtaskqueue_cancel()
407 	error = gtaskqueue_cancel_locked(queue, gtask);  in gtaskqueue_cancel()
408 	TQ_UNLOCK(queue);  in gtaskqueue_cancel()
414 gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain_locked()  argument
416 	while ((gtask->ta_flags & TASK_ENQUEUED) || task_is_running(queue, gtask))  in gtaskqueue_drain_locked()
417 		TQ_SLEEP(queue, gtask, "gtq_drain");  in gtaskqueue_drain_locked()
421 gtaskqueue_drain(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain()  argument
424 	if (!queue->tq_spin)  in gtaskqueue_drain()
427 	TQ_LOCK(queue);  in gtaskqueue_drain()
428 	gtaskqueue_drain_locked(queue, gtask);  in gtaskqueue_drain()
429 	TQ_UNLOCK(queue);  in gtaskqueue_drain()
433 gtaskqueue_drain_all(struct gtaskqueue *queue)  in gtaskqueue_drain_all()  argument
436 	if (!queue->tq_spin)  in gtaskqueue_drain_all()
439 	TQ_LOCK(queue);  in gtaskqueue_drain_all()
440 	gtaskqueue_drain_tq_queue(queue);  in gtaskqueue_drain_all()
441 	gtaskqueue_drain_tq_active(queue);  in gtaskqueue_drain_all()
442 	TQ_UNLOCK(queue);  in gtaskqueue_drain_all()
638 	 * Two passes: first scan for a queue with the least tasks that  in taskqgroup_find()
640 	 * the queue with the least total tasks.  in taskqgroup_find()