xref: /linux/kernel/softirq.c (revision e27ecdd94d81e5bc3d1f68591701db5adb342f0d)
1 /*
2  *	linux/kernel/softirq.c
3  *
4  *	Copyright (C) 1992 Linus Torvalds
5  *
6  *	Distribute under GPLv2.
7  *
8  *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *	Remote softirq infrastructure is by Jens Axboe.
11  */
12 
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/tick.h>
27 
28 #define CREATE_TRACE_POINTS
29 #include <trace/events/irq.h>
30 
31 #include <asm/irq.h>
32 /*
33    - No shared variables, all the data are CPU local.
34    - If a softirq needs serialization, let it serialize itself
35      by its own spinlocks.
36    - Even if softirq is serialized, only local cpu is marked for
37      execution. Hence, we get something sort of weak cpu binding.
38      Though it is still not clear, will it result in better locality
39      or will not.
40 
41    Examples:
42    - NET RX softirq. It is multithreaded and does not require
43      any global serialization.
44    - NET TX softirq. It kicks software netdevice queues, hence
45      it is logically serialized per device, but this serialization
46      is invisible to common code.
47    - Tasklets: serialized wrt itself.
48  */
49 
50 #ifndef __ARCH_IRQ_STAT
51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52 EXPORT_SYMBOL(irq_stat);
53 #endif
54 
55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56 
57 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58 
59 char *softirq_to_name[NR_SOFTIRQS] = {
60 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
61 	"TASKLET", "SCHED", "HRTIMER",	"RCU"
62 };
63 
64 /*
65  * we cannot loop indefinitely here to avoid userspace starvation,
66  * but we also don't want to introduce a worst case 1/HZ latency
67  * to the pending events, so lets the scheduler to balance
68  * the softirq load for us.
69  */
70 void wakeup_softirqd(void)
71 {
72 	/* Interrupts are disabled: no need to stop preemption */
73 	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
74 
75 	if (tsk && tsk->state != TASK_RUNNING)
76 		wake_up_process(tsk);
77 }
78 
79 /*
80  * This one is for softirq.c-internal use,
81  * where hardirqs are disabled legitimately:
82  */
83 #ifdef CONFIG_TRACE_IRQFLAGS
84 static void __local_bh_disable(unsigned long ip)
85 {
86 	unsigned long flags;
87 
88 	WARN_ON_ONCE(in_irq());
89 
90 	raw_local_irq_save(flags);
91 	/*
92 	 * The preempt tracer hooks into add_preempt_count and will break
93 	 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
94 	 * is set and before current->softirq_enabled is cleared.
95 	 * We must manually increment preempt_count here and manually
96 	 * call the trace_preempt_off later.
97 	 */
98 	preempt_count() += SOFTIRQ_OFFSET;
99 	/*
100 	 * Were softirqs turned off above:
101 	 */
102 	if (softirq_count() == SOFTIRQ_OFFSET)
103 		trace_softirqs_off(ip);
104 	raw_local_irq_restore(flags);
105 
106 	if (preempt_count() == SOFTIRQ_OFFSET)
107 		trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
108 }
109 #else /* !CONFIG_TRACE_IRQFLAGS */
110 static inline void __local_bh_disable(unsigned long ip)
111 {
112 	add_preempt_count(SOFTIRQ_OFFSET);
113 	barrier();
114 }
115 #endif /* CONFIG_TRACE_IRQFLAGS */
116 
117 void local_bh_disable(void)
118 {
119 	__local_bh_disable((unsigned long)__builtin_return_address(0));
120 }
121 
122 EXPORT_SYMBOL(local_bh_disable);
123 
124 /*
125  * Special-case - softirqs can safely be enabled in
126  * cond_resched_softirq(), or by __do_softirq(),
127  * without processing still-pending softirqs:
128  */
129 void _local_bh_enable(void)
130 {
131 	WARN_ON_ONCE(in_irq());
132 	WARN_ON_ONCE(!irqs_disabled());
133 
134 	if (softirq_count() == SOFTIRQ_OFFSET)
135 		trace_softirqs_on((unsigned long)__builtin_return_address(0));
136 	sub_preempt_count(SOFTIRQ_OFFSET);
137 }
138 
139 EXPORT_SYMBOL(_local_bh_enable);
140 
141 static inline void _local_bh_enable_ip(unsigned long ip)
142 {
143 	WARN_ON_ONCE(in_irq() || irqs_disabled());
144 #ifdef CONFIG_TRACE_IRQFLAGS
145 	local_irq_disable();
146 #endif
147 	/*
148 	 * Are softirqs going to be turned on now:
149 	 */
150 	if (softirq_count() == SOFTIRQ_OFFSET)
151 		trace_softirqs_on(ip);
152 	/*
153 	 * Keep preemption disabled until we are done with
154 	 * softirq processing:
155  	 */
156  	sub_preempt_count(SOFTIRQ_OFFSET - 1);
157 
158 	if (unlikely(!in_interrupt() && local_softirq_pending()))
159 		do_softirq();
160 
161 	dec_preempt_count();
162 #ifdef CONFIG_TRACE_IRQFLAGS
163 	local_irq_enable();
164 #endif
165 	preempt_check_resched();
166 }
167 
168 void local_bh_enable(void)
169 {
170 	_local_bh_enable_ip((unsigned long)__builtin_return_address(0));
171 }
172 EXPORT_SYMBOL(local_bh_enable);
173 
174 void local_bh_enable_ip(unsigned long ip)
175 {
176 	_local_bh_enable_ip(ip);
177 }
178 EXPORT_SYMBOL(local_bh_enable_ip);
179 
180 /*
181  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
182  * and we fall back to softirqd after that.
183  *
184  * This number has been established via experimentation.
185  * The two things to balance is latency against fairness -
186  * we want to handle softirqs as soon as possible, but they
187  * should not be able to lock up the box.
188  */
189 #define MAX_SOFTIRQ_RESTART 10
190 
191 asmlinkage void __do_softirq(void)
192 {
193 	struct softirq_action *h;
194 	__u32 pending;
195 	int max_restart = MAX_SOFTIRQ_RESTART;
196 	int cpu;
197 
198 	pending = local_softirq_pending();
199 	account_system_vtime(current);
200 
201 	__local_bh_disable((unsigned long)__builtin_return_address(0));
202 	lockdep_softirq_enter();
203 
204 	cpu = smp_processor_id();
205 restart:
206 	/* Reset the pending bitmask before enabling irqs */
207 	set_softirq_pending(0);
208 
209 	local_irq_enable();
210 
211 	h = softirq_vec;
212 
213 	do {
214 		if (pending & 1) {
215 			int prev_count = preempt_count();
216 
217 			trace_softirq_entry(h, softirq_vec);
218 			h->action(h);
219 			trace_softirq_exit(h, softirq_vec);
220 			if (unlikely(prev_count != preempt_count())) {
221 				printk(KERN_ERR "huh, entered softirq %td %s %p"
222 				       "with preempt_count %08x,"
223 				       " exited with %08x?\n", h - softirq_vec,
224 				       softirq_to_name[h - softirq_vec],
225 				       h->action, prev_count, preempt_count());
226 				preempt_count() = prev_count;
227 			}
228 
229 			rcu_bh_qsctr_inc(cpu);
230 		}
231 		h++;
232 		pending >>= 1;
233 	} while (pending);
234 
235 	local_irq_disable();
236 
237 	pending = local_softirq_pending();
238 	if (pending && --max_restart)
239 		goto restart;
240 
241 	if (pending)
242 		wakeup_softirqd();
243 
244 	lockdep_softirq_exit();
245 
246 	account_system_vtime(current);
247 	_local_bh_enable();
248 }
249 
250 #ifndef __ARCH_HAS_DO_SOFTIRQ
251 
252 asmlinkage void do_softirq(void)
253 {
254 	__u32 pending;
255 	unsigned long flags;
256 
257 	if (in_interrupt())
258 		return;
259 
260 	local_irq_save(flags);
261 
262 	pending = local_softirq_pending();
263 
264 	if (pending)
265 		__do_softirq();
266 
267 	local_irq_restore(flags);
268 }
269 
270 #endif
271 
272 /*
273  * Enter an interrupt context.
274  */
275 void irq_enter(void)
276 {
277 	int cpu = smp_processor_id();
278 
279 	rcu_irq_enter();
280 	if (idle_cpu(cpu) && !in_interrupt()) {
281 		__irq_enter();
282 		tick_check_idle(cpu);
283 	} else
284 		__irq_enter();
285 }
286 
287 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
288 # define invoke_softirq()	__do_softirq()
289 #else
290 # define invoke_softirq()	do_softirq()
291 #endif
292 
293 /*
294  * Exit an interrupt context. Process softirqs if needed and possible:
295  */
296 void irq_exit(void)
297 {
298 	account_system_vtime(current);
299 	trace_hardirq_exit();
300 	sub_preempt_count(IRQ_EXIT_OFFSET);
301 	if (!in_interrupt() && local_softirq_pending())
302 		invoke_softirq();
303 
304 #ifdef CONFIG_NO_HZ
305 	/* Make sure that timer wheel updates are propagated */
306 	rcu_irq_exit();
307 	if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
308 		tick_nohz_stop_sched_tick(0);
309 #endif
310 	preempt_enable_no_resched();
311 }
312 
313 /*
314  * This function must run with irqs disabled!
315  */
316 inline void raise_softirq_irqoff(unsigned int nr)
317 {
318 	__raise_softirq_irqoff(nr);
319 
320 	/*
321 	 * If we're in an interrupt or softirq, we're done
322 	 * (this also catches softirq-disabled code). We will
323 	 * actually run the softirq once we return from
324 	 * the irq or softirq.
325 	 *
326 	 * Otherwise we wake up ksoftirqd to make sure we
327 	 * schedule the softirq soon.
328 	 */
329 	if (!in_interrupt())
330 		wakeup_softirqd();
331 }
332 
333 void raise_softirq(unsigned int nr)
334 {
335 	unsigned long flags;
336 
337 	local_irq_save(flags);
338 	raise_softirq_irqoff(nr);
339 	local_irq_restore(flags);
340 }
341 
342 void open_softirq(int nr, void (*action)(struct softirq_action *))
343 {
344 	softirq_vec[nr].action = action;
345 }
346 
347 /* Tasklets */
348 struct tasklet_head
349 {
350 	struct tasklet_struct *head;
351 	struct tasklet_struct **tail;
352 };
353 
354 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
355 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
356 
357 void __tasklet_schedule(struct tasklet_struct *t)
358 {
359 	unsigned long flags;
360 
361 	local_irq_save(flags);
362 	t->next = NULL;
363 	*__get_cpu_var(tasklet_vec).tail = t;
364 	__get_cpu_var(tasklet_vec).tail = &(t->next);
365 	raise_softirq_irqoff(TASKLET_SOFTIRQ);
366 	local_irq_restore(flags);
367 }
368 
369 EXPORT_SYMBOL(__tasklet_schedule);
370 
371 void __tasklet_hi_schedule(struct tasklet_struct *t)
372 {
373 	unsigned long flags;
374 
375 	local_irq_save(flags);
376 	t->next = NULL;
377 	*__get_cpu_var(tasklet_hi_vec).tail = t;
378 	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
379 	raise_softirq_irqoff(HI_SOFTIRQ);
380 	local_irq_restore(flags);
381 }
382 
383 EXPORT_SYMBOL(__tasklet_hi_schedule);
384 
385 static void tasklet_action(struct softirq_action *a)
386 {
387 	struct tasklet_struct *list;
388 
389 	local_irq_disable();
390 	list = __get_cpu_var(tasklet_vec).head;
391 	__get_cpu_var(tasklet_vec).head = NULL;
392 	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
393 	local_irq_enable();
394 
395 	while (list) {
396 		struct tasklet_struct *t = list;
397 
398 		list = list->next;
399 
400 		if (tasklet_trylock(t)) {
401 			if (!atomic_read(&t->count)) {
402 				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
403 					BUG();
404 				t->func(t->data);
405 				tasklet_unlock(t);
406 				continue;
407 			}
408 			tasklet_unlock(t);
409 		}
410 
411 		local_irq_disable();
412 		t->next = NULL;
413 		*__get_cpu_var(tasklet_vec).tail = t;
414 		__get_cpu_var(tasklet_vec).tail = &(t->next);
415 		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
416 		local_irq_enable();
417 	}
418 }
419 
420 static void tasklet_hi_action(struct softirq_action *a)
421 {
422 	struct tasklet_struct *list;
423 
424 	local_irq_disable();
425 	list = __get_cpu_var(tasklet_hi_vec).head;
426 	__get_cpu_var(tasklet_hi_vec).head = NULL;
427 	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
428 	local_irq_enable();
429 
430 	while (list) {
431 		struct tasklet_struct *t = list;
432 
433 		list = list->next;
434 
435 		if (tasklet_trylock(t)) {
436 			if (!atomic_read(&t->count)) {
437 				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
438 					BUG();
439 				t->func(t->data);
440 				tasklet_unlock(t);
441 				continue;
442 			}
443 			tasklet_unlock(t);
444 		}
445 
446 		local_irq_disable();
447 		t->next = NULL;
448 		*__get_cpu_var(tasklet_hi_vec).tail = t;
449 		__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
450 		__raise_softirq_irqoff(HI_SOFTIRQ);
451 		local_irq_enable();
452 	}
453 }
454 
455 
456 void tasklet_init(struct tasklet_struct *t,
457 		  void (*func)(unsigned long), unsigned long data)
458 {
459 	t->next = NULL;
460 	t->state = 0;
461 	atomic_set(&t->count, 0);
462 	t->func = func;
463 	t->data = data;
464 }
465 
466 EXPORT_SYMBOL(tasklet_init);
467 
468 void tasklet_kill(struct tasklet_struct *t)
469 {
470 	if (in_interrupt())
471 		printk("Attempt to kill tasklet from interrupt\n");
472 
473 	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
474 		do {
475 			yield();
476 		} while (test_bit(TASKLET_STATE_SCHED, &t->state));
477 	}
478 	tasklet_unlock_wait(t);
479 	clear_bit(TASKLET_STATE_SCHED, &t->state);
480 }
481 
482 EXPORT_SYMBOL(tasklet_kill);
483 
484 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
485 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
486 
487 static void __local_trigger(struct call_single_data *cp, int softirq)
488 {
489 	struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
490 
491 	list_add_tail(&cp->list, head);
492 
493 	/* Trigger the softirq only if the list was previously empty.  */
494 	if (head->next == &cp->list)
495 		raise_softirq_irqoff(softirq);
496 }
497 
498 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
499 static void remote_softirq_receive(void *data)
500 {
501 	struct call_single_data *cp = data;
502 	unsigned long flags;
503 	int softirq;
504 
505 	softirq = cp->priv;
506 
507 	local_irq_save(flags);
508 	__local_trigger(cp, softirq);
509 	local_irq_restore(flags);
510 }
511 
512 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
513 {
514 	if (cpu_online(cpu)) {
515 		cp->func = remote_softirq_receive;
516 		cp->info = cp;
517 		cp->flags = 0;
518 		cp->priv = softirq;
519 
520 		__smp_call_function_single(cpu, cp, 0);
521 		return 0;
522 	}
523 	return 1;
524 }
525 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
526 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
527 {
528 	return 1;
529 }
530 #endif
531 
532 /**
533  * __send_remote_softirq - try to schedule softirq work on a remote cpu
534  * @cp: private SMP call function data area
535  * @cpu: the remote cpu
536  * @this_cpu: the currently executing cpu
537  * @softirq: the softirq for the work
538  *
539  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
540  * done, the work is instead queued up on the local cpu.
541  *
542  * Interrupts must be disabled.
543  */
544 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
545 {
546 	if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
547 		__local_trigger(cp, softirq);
548 }
549 EXPORT_SYMBOL(__send_remote_softirq);
550 
551 /**
552  * send_remote_softirq - try to schedule softirq work on a remote cpu
553  * @cp: private SMP call function data area
554  * @cpu: the remote cpu
555  * @softirq: the softirq for the work
556  *
557  * Like __send_remote_softirq except that disabling interrupts and
558  * computing the current cpu is done for the caller.
559  */
560 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
561 {
562 	unsigned long flags;
563 	int this_cpu;
564 
565 	local_irq_save(flags);
566 	this_cpu = smp_processor_id();
567 	__send_remote_softirq(cp, cpu, this_cpu, softirq);
568 	local_irq_restore(flags);
569 }
570 EXPORT_SYMBOL(send_remote_softirq);
571 
572 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
573 					       unsigned long action, void *hcpu)
574 {
575 	/*
576 	 * If a CPU goes away, splice its entries to the current CPU
577 	 * and trigger a run of the softirq
578 	 */
579 	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
580 		int cpu = (unsigned long) hcpu;
581 		int i;
582 
583 		local_irq_disable();
584 		for (i = 0; i < NR_SOFTIRQS; i++) {
585 			struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
586 			struct list_head *local_head;
587 
588 			if (list_empty(head))
589 				continue;
590 
591 			local_head = &__get_cpu_var(softirq_work_list[i]);
592 			list_splice_init(head, local_head);
593 			raise_softirq_irqoff(i);
594 		}
595 		local_irq_enable();
596 	}
597 
598 	return NOTIFY_OK;
599 }
600 
601 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
602 	.notifier_call	= remote_softirq_cpu_notify,
603 };
604 
605 void __init softirq_init(void)
606 {
607 	int cpu;
608 
609 	for_each_possible_cpu(cpu) {
610 		int i;
611 
612 		per_cpu(tasklet_vec, cpu).tail =
613 			&per_cpu(tasklet_vec, cpu).head;
614 		per_cpu(tasklet_hi_vec, cpu).tail =
615 			&per_cpu(tasklet_hi_vec, cpu).head;
616 		for (i = 0; i < NR_SOFTIRQS; i++)
617 			INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
618 	}
619 
620 	register_hotcpu_notifier(&remote_softirq_cpu_notifier);
621 
622 	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
623 	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
624 }
625 
626 static int ksoftirqd(void * __bind_cpu)
627 {
628 	set_current_state(TASK_INTERRUPTIBLE);
629 
630 	while (!kthread_should_stop()) {
631 		preempt_disable();
632 		if (!local_softirq_pending()) {
633 			preempt_enable_no_resched();
634 			schedule();
635 			preempt_disable();
636 		}
637 
638 		__set_current_state(TASK_RUNNING);
639 
640 		while (local_softirq_pending()) {
641 			/* Preempt disable stops cpu going offline.
642 			   If already offline, we'll be on wrong CPU:
643 			   don't process */
644 			if (cpu_is_offline((long)__bind_cpu))
645 				goto wait_to_die;
646 			do_softirq();
647 			preempt_enable_no_resched();
648 			cond_resched();
649 			preempt_disable();
650 			rcu_qsctr_inc((long)__bind_cpu);
651 		}
652 		preempt_enable();
653 		set_current_state(TASK_INTERRUPTIBLE);
654 	}
655 	__set_current_state(TASK_RUNNING);
656 	return 0;
657 
658 wait_to_die:
659 	preempt_enable();
660 	/* Wait for kthread_stop */
661 	set_current_state(TASK_INTERRUPTIBLE);
662 	while (!kthread_should_stop()) {
663 		schedule();
664 		set_current_state(TASK_INTERRUPTIBLE);
665 	}
666 	__set_current_state(TASK_RUNNING);
667 	return 0;
668 }
669 
670 #ifdef CONFIG_HOTPLUG_CPU
671 /*
672  * tasklet_kill_immediate is called to remove a tasklet which can already be
673  * scheduled for execution on @cpu.
674  *
675  * Unlike tasklet_kill, this function removes the tasklet
676  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
677  *
678  * When this function is called, @cpu must be in the CPU_DEAD state.
679  */
680 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
681 {
682 	struct tasklet_struct **i;
683 
684 	BUG_ON(cpu_online(cpu));
685 	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
686 
687 	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
688 		return;
689 
690 	/* CPU is dead, so no lock needed. */
691 	for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
692 		if (*i == t) {
693 			*i = t->next;
694 			/* If this was the tail element, move the tail ptr */
695 			if (*i == NULL)
696 				per_cpu(tasklet_vec, cpu).tail = i;
697 			return;
698 		}
699 	}
700 	BUG();
701 }
702 
703 static void takeover_tasklets(unsigned int cpu)
704 {
705 	/* CPU is dead, so no lock needed. */
706 	local_irq_disable();
707 
708 	/* Find end, append list for that CPU. */
709 	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
710 		*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
711 		__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
712 		per_cpu(tasklet_vec, cpu).head = NULL;
713 		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
714 	}
715 	raise_softirq_irqoff(TASKLET_SOFTIRQ);
716 
717 	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
718 		*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
719 		__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
720 		per_cpu(tasklet_hi_vec, cpu).head = NULL;
721 		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
722 	}
723 	raise_softirq_irqoff(HI_SOFTIRQ);
724 
725 	local_irq_enable();
726 }
727 #endif /* CONFIG_HOTPLUG_CPU */
728 
729 static int __cpuinit cpu_callback(struct notifier_block *nfb,
730 				  unsigned long action,
731 				  void *hcpu)
732 {
733 	int hotcpu = (unsigned long)hcpu;
734 	struct task_struct *p;
735 
736 	switch (action) {
737 	case CPU_UP_PREPARE:
738 	case CPU_UP_PREPARE_FROZEN:
739 		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
740 		if (IS_ERR(p)) {
741 			printk("ksoftirqd for %i failed\n", hotcpu);
742 			return NOTIFY_BAD;
743 		}
744 		kthread_bind(p, hotcpu);
745   		per_cpu(ksoftirqd, hotcpu) = p;
746  		break;
747 	case CPU_ONLINE:
748 	case CPU_ONLINE_FROZEN:
749 		wake_up_process(per_cpu(ksoftirqd, hotcpu));
750 		break;
751 #ifdef CONFIG_HOTPLUG_CPU
752 	case CPU_UP_CANCELED:
753 	case CPU_UP_CANCELED_FROZEN:
754 		if (!per_cpu(ksoftirqd, hotcpu))
755 			break;
756 		/* Unbind so it can run.  Fall thru. */
757 		kthread_bind(per_cpu(ksoftirqd, hotcpu),
758 			     cpumask_any(cpu_online_mask));
759 	case CPU_DEAD:
760 	case CPU_DEAD_FROZEN: {
761 		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
762 
763 		p = per_cpu(ksoftirqd, hotcpu);
764 		per_cpu(ksoftirqd, hotcpu) = NULL;
765 		sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
766 		kthread_stop(p);
767 		takeover_tasklets(hotcpu);
768 		break;
769 	}
770 #endif /* CONFIG_HOTPLUG_CPU */
771  	}
772 	return NOTIFY_OK;
773 }
774 
775 static struct notifier_block __cpuinitdata cpu_nfb = {
776 	.notifier_call = cpu_callback
777 };
778 
779 static __init int spawn_ksoftirqd(void)
780 {
781 	void *cpu = (void *)(long)smp_processor_id();
782 	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
783 
784 	BUG_ON(err == NOTIFY_BAD);
785 	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
786 	register_cpu_notifier(&cpu_nfb);
787 	return 0;
788 }
789 early_initcall(spawn_ksoftirqd);
790 
791 #ifdef CONFIG_SMP
792 /*
793  * Call a function on all processors
794  */
795 int on_each_cpu(void (*func) (void *info), void *info, int wait)
796 {
797 	int ret = 0;
798 
799 	preempt_disable();
800 	ret = smp_call_function(func, info, wait);
801 	local_irq_disable();
802 	func(info);
803 	local_irq_enable();
804 	preempt_enable();
805 	return ret;
806 }
807 EXPORT_SYMBOL(on_each_cpu);
808 #endif
809 
810 /*
811  * [ These __weak aliases are kept in a separate compilation unit, so that
812  *   GCC does not inline them incorrectly. ]
813  */
814 
815 int __init __weak early_irq_init(void)
816 {
817 	return 0;
818 }
819 
820 int __init __weak arch_probe_nr_irqs(void)
821 {
822 	return 0;
823 }
824 
825 int __init __weak arch_early_irq_init(void)
826 {
827 	return 0;
828 }
829 
830 int __weak arch_init_chip_data(struct irq_desc *desc, int node)
831 {
832 	return 0;
833 }
834