1 // SPDX-License-Identifier: GPL-2.0-only
7  *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
38    - No shared variables, all the data are CPU local.
39    - If a softirq needs serialization, let it serialize itself
41    - Even if softirq is serialized, only local cpu is marked for
47    - NET RX softirq. It is multithreaded and does not require
49    - NET TX softirq. It kicks software netdevice queues, hence
52    - Tasklets: serialized wrt itself.
71  * but we also don't want to introduce a worst case 1/HZ latency
97  * - count is changed by SOFTIRQ_OFFSET on entering or leaving softirq
100  * - count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
142  * local_bh_blocked() - Check for idle whether BH processing is blocked
166 	if (!current->softirq_disable_cnt) {  in __local_bh_disable_ip()
190 		current->softirq_disable_cnt = newcnt;  in __local_bh_disable_ip()
200 		if (!current->softirq_disable_cnt)  in __local_bh_disable_ip()
204 		current->softirq_disable_cnt += cnt;  in __local_bh_disable_ip()
205 		WARN_ON_ONCE(current->softirq_disable_cnt < 0);  in __local_bh_disable_ip()
223 		DEBUG_LOCKS_WARN_ON(current->softirq_disable_cnt !=  in __local_bh_enable()
228 		if (current->softirq_disable_cnt == cnt)  in __local_bh_enable()
240 		current->softirq_disable_cnt = newcnt;  in __local_bh_enable()
247 		current->softirq_disable_cnt -= cnt;  in __local_bh_enable()
249 		if (unlock && !current->softirq_disable_cnt) {  in __local_bh_enable()
253 			WARN_ON_ONCE(current->softirq_disable_cnt < 0);  in __local_bh_enable()
274 		curcnt = current->softirq_disable_cnt;  in __local_bh_enable_ip()
350  * idle load balancing. If soft interrupts get raised which haven't been
367  * This one is for softirq.c-internal use, where hardirqs are disabled
381 	 * is set and before current->softirq_enabled is cleared.  in __local_bh_disable_ip()
395 		current->preempt_disable_ip = get_lock_parent_ip();  in __local_bh_disable_ip()
417  * Special-case - softirqs can safely be enabled by __do_softirq(),
418  * without processing still-pending softirqs:
443 	__preempt_count_sub(cnt - 1);  in __local_bh_enable_ip()
539  * The two things to balance is latency against fairness -
550  * not miss-qualify lock contexts and miss possible deadlocks.
582 	unsigned long old_flags = current->flags;  in handle_softirqs()
594 	current->flags &= ~PF_MEMALLOC;  in handle_softirqs()
614 		h += softirq_bit - 1;  in handle_softirqs()
616 		vec_nr = h - softirq_vec;  in handle_softirqs()
622 		h->action();  in handle_softirqs()
626 			       vec_nr, softirq_to_name[vec_nr], h->action,  in handle_softirqs()
642 		    --max_restart)  in handle_softirqs()
660  * irq_enter_rcu - Enter an interrupt context with RCU watching
674  * irq_enter - Enter an interrupt context including RCU update
733  * irq_exit_rcu() - Exit an interrupt context without updating RCU
745  * irq_exit - Exit an interrupt context, update RCU and lockdep
766 	 * (this also catches softirq-disabled code). We will  in raise_softirq_irqoff()
802 	struct tasklet_struct *head;  member
809 static void __tasklet_schedule_common(struct tasklet_struct *t,  in __tasklet_schedule_common()  argument
813 	struct tasklet_head *head;  in __tasklet_schedule_common()  local
817 	head = this_cpu_ptr(headp);  in __tasklet_schedule_common()
818 	t->next = NULL;  in __tasklet_schedule_common()
819 	*head->tail = t;  in __tasklet_schedule_common()
820 	head->tail = &(t->next);  in __tasklet_schedule_common()
825 void __tasklet_schedule(struct tasklet_struct *t)  in __tasklet_schedule()  argument
827 	__tasklet_schedule_common(t, &tasklet_vec,  in __tasklet_schedule()
832 void __tasklet_hi_schedule(struct tasklet_struct *t)  in __tasklet_hi_schedule()  argument
834 	__tasklet_schedule_common(t, &tasklet_hi_vec,  in __tasklet_hi_schedule()
839 static bool tasklet_clear_sched(struct tasklet_struct *t)  in tasklet_clear_sched()  argument
841 	if (test_and_clear_wake_up_bit(TASKLET_STATE_SCHED, &t->state))  in tasklet_clear_sched()
845 		  t->use_callback ? "callback" : "func",  in tasklet_clear_sched()
846 		  t->use_callback ? (void *)t->callback : (void *)t->func);  in tasklet_clear_sched()
876 	atomic_inc(&sync_cb->cb_waiters);  in tasklet_callback_cancel_wait_running()
877 	spin_lock(&sync_cb->cb_lock);  in tasklet_callback_cancel_wait_running()
878 	atomic_dec(&sync_cb->cb_waiters);  in tasklet_callback_cancel_wait_running()
879 	spin_unlock(&sync_cb->cb_lock);  in tasklet_callback_cancel_wait_running()
886 	if (atomic_read(&sync_cb->cb_waiters)) {  in tasklet_callback_sync_wait_running()
887 		spin_unlock(&sync_cb->cb_lock);  in tasklet_callback_sync_wait_running()
888 		spin_lock(&sync_cb->cb_lock);  in tasklet_callback_sync_wait_running()
909 	list = tl_head->head;  in tasklet_action_common()
910 	tl_head->head = NULL;  in tasklet_action_common()
911 	tl_head->tail = &tl_head->head;  in tasklet_action_common()
916 		struct tasklet_struct *t = list;  in tasklet_action_common()  local
918 		list = list->next;  in tasklet_action_common()
920 		if (tasklet_trylock(t)) {  in tasklet_action_common()
921 			if (!atomic_read(&t->count)) {  in tasklet_action_common()
922 				if (tasklet_clear_sched(t)) {  in tasklet_action_common()
923 					if (t->use_callback) {  in tasklet_action_common()
924 						trace_tasklet_entry(t, t->callback);  in tasklet_action_common()
925 						t->callback(t);  in tasklet_action_common()
926 						trace_tasklet_exit(t, t->callback);  in tasklet_action_common()
928 						trace_tasklet_entry(t, t->func);  in tasklet_action_common()
929 						t->func(t->data);  in tasklet_action_common()
930 						trace_tasklet_exit(t, t->func);  in tasklet_action_common()
933 				tasklet_unlock(t);  in tasklet_action_common()
937 			tasklet_unlock(t);  in tasklet_action_common()
941 		t->next = NULL;  in tasklet_action_common()
942 		*tl_head->tail = t;  in tasklet_action_common()
943 		tl_head->tail = &t->next;  in tasklet_action_common()
962 void tasklet_setup(struct tasklet_struct *t,  in tasklet_setup()  argument
965 	t->next = NULL;  in tasklet_setup()
966 	t->state = 0;  in tasklet_setup()
967 	atomic_set(&t->count, 0);  in tasklet_setup()
968 	t->callback = callback;  in tasklet_setup()
969 	t->use_callback = true;  in tasklet_setup()
970 	t->data = 0;  in tasklet_setup()
974 void tasklet_init(struct tasklet_struct *t,  in tasklet_init()  argument
977 	t->next = NULL;  in tasklet_init()
978 	t->state = 0;  in tasklet_init()
979 	atomic_set(&t->count, 0);  in tasklet_init()
980 	t->func = func;  in tasklet_init()
981 	t->use_callback = false;  in tasklet_init()
982 	t->data = data;  in tasklet_init()
991 void tasklet_unlock_spin_wait(struct tasklet_struct *t)  in tasklet_unlock_spin_wait()  argument
993 	while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {  in tasklet_unlock_spin_wait()
1009 void tasklet_kill(struct tasklet_struct *t)  in tasklet_kill()  argument
1014 	wait_on_bit_lock(&t->state, TASKLET_STATE_SCHED, TASK_UNINTERRUPTIBLE);  in tasklet_kill()
1016 	tasklet_unlock_wait(t);  in tasklet_kill()
1017 	tasklet_clear_sched(t);  in tasklet_kill()
1022 void tasklet_unlock(struct tasklet_struct *t)  in tasklet_unlock()  argument
1024 	clear_and_wake_up_bit(TASKLET_STATE_RUN, &t->state);  in tasklet_unlock()
1028 void tasklet_unlock_wait(struct tasklet_struct *t)  in tasklet_unlock_wait()  argument
1030 	wait_on_bit(&t->state, TASKLET_STATE_RUN, TASK_UNINTERRUPTIBLE);  in tasklet_unlock_wait()
1041 			&per_cpu(tasklet_vec, cpu).head;  in softirq_init()
1043 			&per_cpu(tasklet_hi_vec, cpu).head;  in softirq_init()
1080 	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {  in takeover_tasklets()
1081 		*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;  in takeover_tasklets()
1083 		per_cpu(tasklet_vec, cpu).head = NULL;  in takeover_tasklets()
1084 		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;  in takeover_tasklets()
1088 	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {  in takeover_tasklets()
1089 		*__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;  in takeover_tasklets()
1091 		per_cpu(tasklet_hi_vec, cpu).head = NULL;  in takeover_tasklets()
1092 		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;  in takeover_tasklets()