1 /* Modified by Broadcom Corp. Portions Copyright (c) Broadcom Corp, 2012. */
3 * linux/kernel/softirq.c
5 * Copyright (C) 1992 Linus Torvalds
7 * Distribute under GPLv2.
9 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
11 * Remote softirq infrastructure is by Jens Axboe.
14 #include <linux/module.h>
15 #include <linux/kernel_stat.h>
16 #include <linux/interrupt.h>
17 #include <linux/init.h>
19 #include <linux/notifier.h>
20 #include <linux/percpu.h>
21 #include <linux/cpu.h>
22 #include <linux/freezer.h>
23 #include <linux/kthread.h>
24 #include <linux/rcupdate.h>
25 #include <linux/ftrace.h>
26 #include <linux/smp.h>
27 #include <linux/tick.h>
29 #define CREATE_TRACE_POINTS
30 #include <trace/events/irq.h>
34 #if defined(CONFIG_BUZZZ)
35 #include <asm/buzzz.h>
36 #endif /* CONFIG_BUZZZ */
42 - No shared variables, all the data are CPU local.
43 - If a softirq needs serialization, let it serialize itself
45 - Even if softirq is serialized, only local cpu is marked for
46 execution. Hence, we get something sort of weak cpu binding.
47 Though it is still not clear, will it result in better locality
51 - NET RX softirq. It is multithreaded and does not require
52 any global serialization.
53 - NET TX softirq. It kicks software netdevice queues, hence
54 it is logically serialized per device, but this serialization
55 is invisible to common code.
56 - Tasklets: serialized wrt itself.
59 #ifndef __ARCH_IRQ_STAT
60 irq_cpustat_t irq_stat
[NR_CPUS
] ____cacheline_aligned
;
61 EXPORT_SYMBOL(irq_stat
);
64 static struct softirq_action softirq_vec
[NR_SOFTIRQS
] __cacheline_aligned_in_smp
;
66 static DEFINE_PER_CPU(struct task_struct
*, ksoftirqd
);
68 char *softirq_to_name
[NR_SOFTIRQS
] = {
69 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
70 "TASKLET", "SCHED", "HRTIMER", "RCU"
74 * we cannot loop indefinitely here to avoid userspace starvation,
75 * but we also don't want to introduce a worst case 1/HZ latency
76 * to the pending events, so lets the scheduler to balance
77 * the softirq load for us.
79 void wakeup_softirqd(void)
81 /* Interrupts are disabled: no need to stop preemption */
82 struct task_struct
*tsk
= __get_cpu_var(ksoftirqd
);
84 if (tsk
&& tsk
->state
!= TASK_RUNNING
)
89 * This one is for softirq.c-internal use,
90 * where hardirqs are disabled legitimately:
92 #ifdef CONFIG_TRACE_IRQFLAGS
93 static void __local_bh_disable(unsigned long ip
)
97 WARN_ON_ONCE(in_irq());
99 raw_local_irq_save(flags
);
101 * The preempt tracer hooks into add_preempt_count and will break
102 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
103 * is set and before current->softirq_enabled is cleared.
104 * We must manually increment preempt_count here and manually
105 * call the trace_preempt_off later.
107 preempt_count() += SOFTIRQ_OFFSET
;
109 * Were softirqs turned off above:
111 if (softirq_count() == SOFTIRQ_OFFSET
)
112 trace_softirqs_off(ip
);
113 raw_local_irq_restore(flags
);
115 if (preempt_count() == SOFTIRQ_OFFSET
)
116 trace_preempt_off(CALLER_ADDR0
, get_parent_ip(CALLER_ADDR1
));
118 #else /* !CONFIG_TRACE_IRQFLAGS */
119 static inline void __local_bh_disable(unsigned long ip
)
121 add_preempt_count(SOFTIRQ_OFFSET
);
124 #endif /* CONFIG_TRACE_IRQFLAGS */
126 void BCMFASTPATH
local_bh_disable(void)
128 __local_bh_disable((unsigned long)__builtin_return_address(0));
131 EXPORT_SYMBOL(local_bh_disable
);
134 * Special-case - softirqs can safely be enabled in
135 * cond_resched_softirq(), or by __do_softirq(),
136 * without processing still-pending softirqs:
138 void _local_bh_enable(void)
141 WARN_ON_ONCE(in_irq());
142 WARN_ON_ONCE(!irqs_disabled());
145 if (softirq_count() == SOFTIRQ_OFFSET
)
146 trace_softirqs_on((unsigned long)__builtin_return_address(0));
147 sub_preempt_count(SOFTIRQ_OFFSET
);
150 EXPORT_SYMBOL(_local_bh_enable
);
152 static inline void _local_bh_enable_ip(unsigned long ip
)
155 WARN_ON_ONCE(in_irq() || irqs_disabled());
157 #ifdef CONFIG_TRACE_IRQFLAGS
161 * Are softirqs going to be turned on now:
163 if (softirq_count() == SOFTIRQ_OFFSET
)
164 trace_softirqs_on(ip
);
166 * Keep preemption disabled until we are done with
167 * softirq processing:
169 sub_preempt_count(SOFTIRQ_OFFSET
- 1);
171 if (unlikely(!in_interrupt() && local_softirq_pending()))
175 #ifdef CONFIG_TRACE_IRQFLAGS
178 preempt_check_resched();
181 void BCMFASTPATH
local_bh_enable(void)
183 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
185 EXPORT_SYMBOL(local_bh_enable
);
187 void local_bh_enable_ip(unsigned long ip
)
189 _local_bh_enable_ip(ip
);
191 EXPORT_SYMBOL(local_bh_enable_ip
);
194 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
195 * and we fall back to softirqd after that.
197 * This number has been established via experimentation.
198 * The two things to balance is latency against fairness -
199 * we want to handle softirqs as soon as possible, but they
200 * should not be able to lock up the box.
202 #define MAX_SOFTIRQ_RESTART 10
204 asmlinkage
void BCMFASTPATH
__do_softirq(void)
206 struct softirq_action
*h
;
208 int max_restart
= MAX_SOFTIRQ_RESTART
;
211 pending
= local_softirq_pending();
212 account_system_vtime(current
);
214 __local_bh_disable((unsigned long)__builtin_return_address(0));
215 lockdep_softirq_enter();
217 cpu
= smp_processor_id();
219 /* Reset the pending bitmask before enabling irqs */
220 set_softirq_pending(0);
228 int prev_count
= preempt_count();
229 kstat_incr_softirqs_this_cpu(h
- softirq_vec
);
231 trace_softirq_entry(h
, softirq_vec
);
233 #if defined(BUZZZ_KEVT_LVL) && (BUZZZ_KEVT_LVL >= 1)
234 buzzz_kevt_log1(BUZZZ_KEVT_ID_SIRQ_ENTRY
, (int)h
->action
);
235 #endif /* BUZZZ_KEVT_LVL */
239 #if defined(BUZZZ_KEVT_LVL) && (BUZZZ_KEVT_LVL >= 1)
240 buzzz_kevt_log1(BUZZZ_KEVT_ID_SIRQ_EXIT
, (int)h
->action
);
241 #endif /* BUZZZ_KEVT_LVL */
243 trace_softirq_exit(h
, softirq_vec
);
244 if (unlikely(prev_count
!= preempt_count())) {
245 printk(KERN_ERR
"huh, entered softirq %td %s %p"
246 "with preempt_count %08x,"
247 " exited with %08x?\n", h
- softirq_vec
,
248 softirq_to_name
[h
- softirq_vec
],
249 h
->action
, prev_count
, preempt_count());
250 preempt_count() = prev_count
;
261 pending
= local_softirq_pending();
262 if (pending
&& --max_restart
)
268 lockdep_softirq_exit();
270 account_system_vtime(current
);
274 #ifndef __ARCH_HAS_DO_SOFTIRQ
276 asmlinkage
void do_softirq(void)
284 local_irq_save(flags
);
286 pending
= local_softirq_pending();
291 local_irq_restore(flags
);
297 * Enter an interrupt context.
301 int cpu
= smp_processor_id();
304 if (idle_cpu(cpu
) && !in_interrupt()) {
306 tick_check_idle(cpu
);
311 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
312 # define invoke_softirq() __do_softirq()
314 # define invoke_softirq() do_softirq()
318 * Exit an interrupt context. Process softirqs if needed and possible:
322 account_system_vtime(current
);
323 trace_hardirq_exit();
324 sub_preempt_count(IRQ_EXIT_OFFSET
);
325 if (!in_interrupt() && local_softirq_pending())
330 /* Make sure that timer wheel updates are propagated */
331 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
332 tick_nohz_stop_sched_tick(0);
334 preempt_enable_no_resched();
338 * This function must run with irqs disabled!
340 inline void raise_softirq_irqoff(unsigned int nr
)
342 __raise_softirq_irqoff(nr
);
345 * If we're in an interrupt or softirq, we're done
346 * (this also catches softirq-disabled code). We will
347 * actually run the softirq once we return from
348 * the irq or softirq.
350 * Otherwise we wake up ksoftirqd to make sure we
351 * schedule the softirq soon.
357 void raise_softirq(unsigned int nr
)
361 local_irq_save(flags
);
362 raise_softirq_irqoff(nr
);
363 local_irq_restore(flags
);
366 void open_softirq(int nr
, void (*action
)(struct softirq_action
*))
368 softirq_vec
[nr
].action
= action
;
376 struct tasklet_struct
*head
;
377 struct tasklet_struct
**tail
;
380 static DEFINE_PER_CPU(struct tasklet_head
, tasklet_vec
);
381 static DEFINE_PER_CPU(struct tasklet_head
, tasklet_hi_vec
);
383 void __tasklet_schedule(struct tasklet_struct
*t
)
387 local_irq_save(flags
);
389 *__get_cpu_var(tasklet_vec
).tail
= t
;
390 __get_cpu_var(tasklet_vec
).tail
= &(t
->next
);
391 raise_softirq_irqoff(TASKLET_SOFTIRQ
);
392 local_irq_restore(flags
);
395 EXPORT_SYMBOL(__tasklet_schedule
);
397 void __tasklet_hi_schedule(struct tasklet_struct
*t
)
401 local_irq_save(flags
);
403 *__get_cpu_var(tasklet_hi_vec
).tail
= t
;
404 __get_cpu_var(tasklet_hi_vec
).tail
= &(t
->next
);
405 raise_softirq_irqoff(HI_SOFTIRQ
);
406 local_irq_restore(flags
);
409 EXPORT_SYMBOL(__tasklet_hi_schedule
);
411 void __tasklet_hi_schedule_first(struct tasklet_struct
*t
)
413 BUG_ON(!irqs_disabled());
415 t
->next
= __get_cpu_var(tasklet_hi_vec
).head
;
416 __get_cpu_var(tasklet_hi_vec
).head
= t
;
417 __raise_softirq_irqoff(HI_SOFTIRQ
);
420 EXPORT_SYMBOL(__tasklet_hi_schedule_first
);
422 static void tasklet_action(struct softirq_action
*a
)
424 struct tasklet_struct
*list
;
427 list
= __get_cpu_var(tasklet_vec
).head
;
428 __get_cpu_var(tasklet_vec
).head
= NULL
;
429 __get_cpu_var(tasklet_vec
).tail
= &__get_cpu_var(tasklet_vec
).head
;
433 struct tasklet_struct
*t
= list
;
437 if (tasklet_trylock(t
)) {
438 if (!atomic_read(&t
->count
)) {
439 if (!test_and_clear_bit(TASKLET_STATE_SCHED
, &t
->state
))
450 *__get_cpu_var(tasklet_vec
).tail
= t
;
451 __get_cpu_var(tasklet_vec
).tail
= &(t
->next
);
452 __raise_softirq_irqoff(TASKLET_SOFTIRQ
);
457 static void tasklet_hi_action(struct softirq_action
*a
)
459 struct tasklet_struct
*list
;
462 list
= __get_cpu_var(tasklet_hi_vec
).head
;
463 __get_cpu_var(tasklet_hi_vec
).head
= NULL
;
464 __get_cpu_var(tasklet_hi_vec
).tail
= &__get_cpu_var(tasklet_hi_vec
).head
;
468 struct tasklet_struct
*t
= list
;
472 if (tasklet_trylock(t
)) {
473 if (!atomic_read(&t
->count
)) {
474 if (!test_and_clear_bit(TASKLET_STATE_SCHED
, &t
->state
))
485 *__get_cpu_var(tasklet_hi_vec
).tail
= t
;
486 __get_cpu_var(tasklet_hi_vec
).tail
= &(t
->next
);
487 __raise_softirq_irqoff(HI_SOFTIRQ
);
493 void tasklet_init(struct tasklet_struct
*t
,
494 void (*func
)(unsigned long), unsigned long data
)
498 atomic_set(&t
->count
, 0);
503 EXPORT_SYMBOL(tasklet_init
);
505 void tasklet_kill(struct tasklet_struct
*t
)
508 printk("Attempt to kill tasklet from interrupt\n");
510 while (test_and_set_bit(TASKLET_STATE_SCHED
, &t
->state
)) {
513 } while (test_bit(TASKLET_STATE_SCHED
, &t
->state
));
515 tasklet_unlock_wait(t
);
516 clear_bit(TASKLET_STATE_SCHED
, &t
->state
);
519 EXPORT_SYMBOL(tasklet_kill
);
526 * The trampoline is called when the hrtimer expires. It schedules a tasklet
527 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
528 * hrtimer callback, but from softirq context.
530 static enum hrtimer_restart
__hrtimer_tasklet_trampoline(struct hrtimer
*timer
)
532 struct tasklet_hrtimer
*ttimer
=
533 container_of(timer
, struct tasklet_hrtimer
, timer
);
535 tasklet_hi_schedule(&ttimer
->tasklet
);
536 return HRTIMER_NORESTART
;
540 * Helper function which calls the hrtimer callback from
541 * tasklet/softirq context
543 static void __tasklet_hrtimer_trampoline(unsigned long data
)
545 struct tasklet_hrtimer
*ttimer
= (void *)data
;
546 enum hrtimer_restart restart
;
548 restart
= ttimer
->function(&ttimer
->timer
);
549 if (restart
!= HRTIMER_NORESTART
)
550 hrtimer_restart(&ttimer
->timer
);
554 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
555 * @ttimer: tasklet_hrtimer which is initialized
556 * @function: hrtimer callback funtion which gets called from softirq context
557 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
558 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
560 void tasklet_hrtimer_init(struct tasklet_hrtimer
*ttimer
,
561 enum hrtimer_restart (*function
)(struct hrtimer
*),
562 clockid_t which_clock
, enum hrtimer_mode mode
)
564 hrtimer_init(&ttimer
->timer
, which_clock
, mode
);
565 ttimer
->timer
.function
= __hrtimer_tasklet_trampoline
;
566 tasklet_init(&ttimer
->tasklet
, __tasklet_hrtimer_trampoline
,
567 (unsigned long)ttimer
);
568 ttimer
->function
= function
;
570 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init
);
573 * Remote softirq bits
576 DEFINE_PER_CPU(struct list_head
[NR_SOFTIRQS
], softirq_work_list
);
577 EXPORT_PER_CPU_SYMBOL(softirq_work_list
);
579 static void __local_trigger(struct call_single_data
*cp
, int softirq
)
581 struct list_head
*head
= &__get_cpu_var(softirq_work_list
[softirq
]);
583 list_add_tail(&cp
->list
, head
);
585 /* Trigger the softirq only if the list was previously empty. */
586 if (head
->next
== &cp
->list
)
587 raise_softirq_irqoff(softirq
);
590 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
591 static void remote_softirq_receive(void *data
)
593 struct call_single_data
*cp
= data
;
599 local_irq_save(flags
);
600 __local_trigger(cp
, softirq
);
601 local_irq_restore(flags
);
604 static int __try_remote_softirq(struct call_single_data
*cp
, int cpu
, int softirq
)
606 if (cpu_online(cpu
)) {
607 cp
->func
= remote_softirq_receive
;
612 __smp_call_function_single(cpu
, cp
, 0);
617 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
618 static int __try_remote_softirq(struct call_single_data
*cp
, int cpu
, int softirq
)
625 * __send_remote_softirq - try to schedule softirq work on a remote cpu
626 * @cp: private SMP call function data area
627 * @cpu: the remote cpu
628 * @this_cpu: the currently executing cpu
629 * @softirq: the softirq for the work
631 * Attempt to schedule softirq work on a remote cpu. If this cannot be
632 * done, the work is instead queued up on the local cpu.
634 * Interrupts must be disabled.
636 void __send_remote_softirq(struct call_single_data
*cp
, int cpu
, int this_cpu
, int softirq
)
638 if (cpu
== this_cpu
|| __try_remote_softirq(cp
, cpu
, softirq
))
639 __local_trigger(cp
, softirq
);
641 EXPORT_SYMBOL(__send_remote_softirq
);
644 * send_remote_softirq - try to schedule softirq work on a remote cpu
645 * @cp: private SMP call function data area
646 * @cpu: the remote cpu
647 * @softirq: the softirq for the work
649 * Like __send_remote_softirq except that disabling interrupts and
650 * computing the current cpu is done for the caller.
652 void send_remote_softirq(struct call_single_data
*cp
, int cpu
, int softirq
)
657 local_irq_save(flags
);
658 this_cpu
= smp_processor_id();
659 __send_remote_softirq(cp
, cpu
, this_cpu
, softirq
);
660 local_irq_restore(flags
);
662 EXPORT_SYMBOL(send_remote_softirq
);
664 static int __cpuinit
remote_softirq_cpu_notify(struct notifier_block
*self
,
665 unsigned long action
, void *hcpu
)
668 * If a CPU goes away, splice its entries to the current CPU
669 * and trigger a run of the softirq
671 if (action
== CPU_DEAD
|| action
== CPU_DEAD_FROZEN
) {
672 int cpu
= (unsigned long) hcpu
;
676 for (i
= 0; i
< NR_SOFTIRQS
; i
++) {
677 struct list_head
*head
= &per_cpu(softirq_work_list
[i
], cpu
);
678 struct list_head
*local_head
;
680 if (list_empty(head
))
683 local_head
= &__get_cpu_var(softirq_work_list
[i
]);
684 list_splice_init(head
, local_head
);
685 raise_softirq_irqoff(i
);
693 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier
= {
694 .notifier_call
= remote_softirq_cpu_notify
,
697 void __init
softirq_init(void)
701 for_each_possible_cpu(cpu
) {
704 per_cpu(tasklet_vec
, cpu
).tail
=
705 &per_cpu(tasklet_vec
, cpu
).head
;
706 per_cpu(tasklet_hi_vec
, cpu
).tail
=
707 &per_cpu(tasklet_hi_vec
, cpu
).head
;
708 for (i
= 0; i
< NR_SOFTIRQS
; i
++)
709 INIT_LIST_HEAD(&per_cpu(softirq_work_list
[i
], cpu
));
712 register_hotcpu_notifier(&remote_softirq_cpu_notifier
);
714 open_softirq(TASKLET_SOFTIRQ
, tasklet_action
);
715 open_softirq(HI_SOFTIRQ
, tasklet_hi_action
);
718 static int run_ksoftirqd(void * __bind_cpu
)
720 set_current_state(TASK_INTERRUPTIBLE
);
722 while (!kthread_should_stop()) {
724 if (!local_softirq_pending()) {
725 preempt_enable_no_resched();
730 __set_current_state(TASK_RUNNING
);
732 while (local_softirq_pending()) {
733 /* Preempt disable stops cpu going offline.
734 If already offline, we'll be on wrong CPU:
736 if (cpu_is_offline((long)__bind_cpu
))
739 preempt_enable_no_resched();
742 rcu_note_context_switch((long)__bind_cpu
);
745 set_current_state(TASK_INTERRUPTIBLE
);
747 __set_current_state(TASK_RUNNING
);
752 /* Wait for kthread_stop */
753 set_current_state(TASK_INTERRUPTIBLE
);
754 while (!kthread_should_stop()) {
756 set_current_state(TASK_INTERRUPTIBLE
);
758 __set_current_state(TASK_RUNNING
);
762 #ifdef CONFIG_HOTPLUG_CPU
764 * tasklet_kill_immediate is called to remove a tasklet which can already be
765 * scheduled for execution on @cpu.
767 * Unlike tasklet_kill, this function removes the tasklet
768 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
770 * When this function is called, @cpu must be in the CPU_DEAD state.
772 void tasklet_kill_immediate(struct tasklet_struct
*t
, unsigned int cpu
)
774 struct tasklet_struct
**i
;
776 BUG_ON(cpu_online(cpu
));
777 BUG_ON(test_bit(TASKLET_STATE_RUN
, &t
->state
));
779 if (!test_bit(TASKLET_STATE_SCHED
, &t
->state
))
782 /* CPU is dead, so no lock needed. */
783 for (i
= &per_cpu(tasklet_vec
, cpu
).head
; *i
; i
= &(*i
)->next
) {
786 /* If this was the tail element, move the tail ptr */
788 per_cpu(tasklet_vec
, cpu
).tail
= i
;
795 static void takeover_tasklets(unsigned int cpu
)
797 /* CPU is dead, so no lock needed. */
800 /* Find end, append list for that CPU. */
801 if (&per_cpu(tasklet_vec
, cpu
).head
!= per_cpu(tasklet_vec
, cpu
).tail
) {
802 *(__get_cpu_var(tasklet_vec
).tail
) = per_cpu(tasklet_vec
, cpu
).head
;
803 __get_cpu_var(tasklet_vec
).tail
= per_cpu(tasklet_vec
, cpu
).tail
;
804 per_cpu(tasklet_vec
, cpu
).head
= NULL
;
805 per_cpu(tasklet_vec
, cpu
).tail
= &per_cpu(tasklet_vec
, cpu
).head
;
807 raise_softirq_irqoff(TASKLET_SOFTIRQ
);
809 if (&per_cpu(tasklet_hi_vec
, cpu
).head
!= per_cpu(tasklet_hi_vec
, cpu
).tail
) {
810 *__get_cpu_var(tasklet_hi_vec
).tail
= per_cpu(tasklet_hi_vec
, cpu
).head
;
811 __get_cpu_var(tasklet_hi_vec
).tail
= per_cpu(tasklet_hi_vec
, cpu
).tail
;
812 per_cpu(tasklet_hi_vec
, cpu
).head
= NULL
;
813 per_cpu(tasklet_hi_vec
, cpu
).tail
= &per_cpu(tasklet_hi_vec
, cpu
).head
;
815 raise_softirq_irqoff(HI_SOFTIRQ
);
819 #endif /* CONFIG_HOTPLUG_CPU */
821 static int __cpuinit
cpu_callback(struct notifier_block
*nfb
,
822 unsigned long action
,
825 int hotcpu
= (unsigned long)hcpu
;
826 struct task_struct
*p
;
830 case CPU_UP_PREPARE_FROZEN
:
831 p
= kthread_create(run_ksoftirqd
, hcpu
, "ksoftirqd/%d", hotcpu
);
833 printk("ksoftirqd for %i failed\n", hotcpu
);
834 return notifier_from_errno(PTR_ERR(p
));
836 kthread_bind(p
, hotcpu
);
837 per_cpu(ksoftirqd
, hotcpu
) = p
;
840 case CPU_ONLINE_FROZEN
:
841 wake_up_process(per_cpu(ksoftirqd
, hotcpu
));
843 #ifdef CONFIG_HOTPLUG_CPU
844 case CPU_UP_CANCELED
:
845 case CPU_UP_CANCELED_FROZEN
:
846 if (!per_cpu(ksoftirqd
, hotcpu
))
848 /* Unbind so it can run. Fall thru. */
849 kthread_bind(per_cpu(ksoftirqd
, hotcpu
),
850 cpumask_any(cpu_online_mask
));
852 case CPU_DEAD_FROZEN
: {
853 struct sched_param param
= { .sched_priority
= MAX_RT_PRIO
-1 };
855 p
= per_cpu(ksoftirqd
, hotcpu
);
856 per_cpu(ksoftirqd
, hotcpu
) = NULL
;
857 sched_setscheduler_nocheck(p
, SCHED_FIFO
, ¶m
);
859 takeover_tasklets(hotcpu
);
862 #endif /* CONFIG_HOTPLUG_CPU */
867 static struct notifier_block __cpuinitdata cpu_nfb
= {
868 .notifier_call
= cpu_callback
871 static __init
int spawn_ksoftirqd(void)
873 void *cpu
= (void *)(long)smp_processor_id();
874 int err
= cpu_callback(&cpu_nfb
, CPU_UP_PREPARE
, cpu
);
876 BUG_ON(err
!= NOTIFY_OK
);
877 cpu_callback(&cpu_nfb
, CPU_ONLINE
, cpu
);
878 register_cpu_notifier(&cpu_nfb
);
881 early_initcall(spawn_ksoftirqd
);
885 * Call a function on all processors
887 int on_each_cpu(void (*func
) (void *info
), void *info
, int wait
)
892 ret
= smp_call_function(func
, info
, wait
);
899 EXPORT_SYMBOL(on_each_cpu
);
903 * [ These __weak aliases are kept in a separate compilation unit, so that
904 * GCC does not inline them incorrectly. ]
907 int __init __weak
early_irq_init(void)
912 int __init __weak
arch_probe_nr_irqs(void)
917 int __init __weak
arch_early_irq_init(void)
922 int __weak
arch_init_chip_data(struct irq_desc
*desc
, int node
)