51 static int	task_is_running(struct gtaskqueue *queue, struct gtask *gtask);
52 static void	gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask);
57 	struct gtask		*tb_running;
65 	STAILQ_HEAD(, gtask)	tq_queue;
107 gtask_dump(struct gtask *gtask)  in gtask_dump()  argument
110 	       gtask, gtask->ta_flags, gtask->ta_priority, gtask->ta_func, gtask->ta_context);  in gtask_dump()
191 	struct gtask *gtask = &grouptask->gt_task;  in grouptask_block()  local
195 		gtask_dump(gtask);  in grouptask_block()
200 	gtask->ta_flags |= TASK_NOENQUEUE;  in grouptask_block()
201   	gtaskqueue_drain_locked(queue, gtask);  in grouptask_block()
209 	struct gtask *gtask = &grouptask->gt_task;  in grouptask_unblock()  local
213 		gtask_dump(gtask);  in grouptask_unblock()
218 	gtask->ta_flags &= ~TASK_NOENQUEUE;  in grouptask_unblock()
223 grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask)  in grouptaskqueue_enqueue()  argument
227 		gtask_dump(gtask);  in grouptaskqueue_enqueue()
232 	if (gtask->ta_flags & TASK_ENQUEUED) {  in grouptaskqueue_enqueue()
236 	if (gtask->ta_flags & TASK_NOENQUEUE) {  in grouptaskqueue_enqueue()
240 	STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link);  in grouptaskqueue_enqueue()
241 	gtask->ta_flags |= TASK_ENQUEUED;  in grouptaskqueue_enqueue()
261 	struct gtask t_barrier;  in gtaskqueue_drain_tq_queue()
344 	struct gtask *gtask;  in gtaskqueue_run_locked()  local
353 	while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) {  in gtaskqueue_run_locked()
355 		gtask->ta_flags &= ~TASK_ENQUEUED;  in gtaskqueue_run_locked()
356 		tb.tb_running = gtask;  in gtaskqueue_run_locked()
360 		KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL"));  in gtaskqueue_run_locked()
361 		if (!in_net_epoch && TASK_IS_NET(gtask)) {  in gtaskqueue_run_locked()
364 		} else if (in_net_epoch && !TASK_IS_NET(gtask)) {  in gtaskqueue_run_locked()
368 		gtask->ta_func(gtask->ta_context);  in gtaskqueue_run_locked()
371 		wakeup(gtask);  in gtaskqueue_run_locked()
379 task_is_running(struct gtaskqueue *queue, struct gtask *gtask)  in task_is_running()  argument
385 		if (tb->tb_running == gtask)  in task_is_running()
392 gtaskqueue_cancel_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel_locked()  argument
395 	if (gtask->ta_flags & TASK_ENQUEUED)  in gtaskqueue_cancel_locked()
396 		STAILQ_REMOVE(&queue->tq_queue, gtask, gtask, ta_link);  in gtaskqueue_cancel_locked()
397 	gtask->ta_flags &= ~TASK_ENQUEUED;  in gtaskqueue_cancel_locked()
398 	return (task_is_running(queue, gtask) ? EBUSY : 0);  in gtaskqueue_cancel_locked()
402 gtaskqueue_cancel(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_cancel()  argument
407 	error = gtaskqueue_cancel_locked(queue, gtask);  in gtaskqueue_cancel()
414 gtaskqueue_drain_locked(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain_locked()  argument
416 	while ((gtask->ta_flags & TASK_ENQUEUED) || task_is_running(queue, gtask))  in gtaskqueue_drain_locked()
417 		TQ_SLEEP(queue, gtask, "gtq_drain");  in gtaskqueue_drain_locked()
421 gtaskqueue_drain(struct gtaskqueue *queue, struct gtask *gtask)  in gtaskqueue_drain()  argument
428 	gtaskqueue_drain_locked(queue, gtask);  in gtaskqueue_drain()
604 	struct gtask bt_task;
666 taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask,  in taskqgroup_attach()  argument
674 	gtask->gt_uniq = uniq;  in taskqgroup_attach()
675 	snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask");  in taskqgroup_attach()
676 	gtask->gt_dev = dev;  in taskqgroup_attach()
677 	gtask->gt_irq = irq;  in taskqgroup_attach()
678 	gtask->gt_cpu = -1;  in taskqgroup_attach()
682 	LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list);  in taskqgroup_attach()
683 	gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq;  in taskqgroup_attach()
686 		gtask->gt_cpu = cpu;  in taskqgroup_attach()
691 			    __func__, gtask->gt_name, error);  in taskqgroup_attach()
697 taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask,  in taskqgroup_attach_cpu()  argument
702 	gtask->gt_uniq = uniq;  in taskqgroup_attach_cpu()
703 	snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask");  in taskqgroup_attach_cpu()
704 	gtask->gt_dev = dev;  in taskqgroup_attach_cpu()
705 	gtask->gt_irq = irq;  in taskqgroup_attach_cpu()
706 	gtask->gt_cpu = cpu;  in taskqgroup_attach_cpu()
715 		printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu);  in taskqgroup_attach_cpu()
719 	LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list);  in taskqgroup_attach_cpu()
720 	gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq;  in taskqgroup_attach_cpu()
728 			    __func__, gtask->gt_name, error);  in taskqgroup_attach_cpu()
734 taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask)  in taskqgroup_detach()  argument
738 	grouptask_block(gtask);  in taskqgroup_detach()
741 		if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue)  in taskqgroup_detach()
744 		panic("%s: task %s not in group", __func__, gtask->gt_name);  in taskqgroup_detach()
746 	LIST_REMOVE(gtask, gt_list);  in taskqgroup_detach()
748 	gtask->gt_taskqueue = NULL;  in taskqgroup_detach()
749 	gtask->gt_task.ta_flags &= ~TASK_NOENQUEUE;  in taskqgroup_detach()
755 	struct taskq_bind_task *gtask;  in taskqgroup_binder()  local
759 	gtask = ctx;  in taskqgroup_binder()
761 	CPU_SET(gtask->bt_cpuid, &mask);  in taskqgroup_binder()
764 	sched_bind(curthread, gtask->bt_cpuid);  in taskqgroup_binder()
769 	free(gtask, M_DEVBUF);  in taskqgroup_binder()
775 	struct taskq_bind_task *gtask;  in taskqgroup_bind()  local
786 		gtask = malloc(sizeof(*gtask), M_DEVBUF, M_WAITOK);  in taskqgroup_bind()
787 		GTASK_INIT(&gtask->bt_task, 0, 0, taskqgroup_binder, gtask);  in taskqgroup_bind()
788 		gtask->bt_cpuid = qgroup->tqg_queue[i].tgc_cpu;  in taskqgroup_bind()
790 		    &gtask->bt_task);  in taskqgroup_bind()