Driver core: convert fb code to use struct device
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / kernel / softirq.c
blobbf25015dce162a541059c051567c3569069985b9
1 /*
2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
7 */
9 #include <linux/module.h>
10 #include <linux/kernel_stat.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
13 #include <linux/mm.h>
14 #include <linux/notifier.h>
15 #include <linux/percpu.h>
16 #include <linux/cpu.h>
17 #include <linux/kthread.h>
18 #include <linux/rcupdate.h>
19 #include <linux/smp.h>
21 #include <asm/irq.h>
23 - No shared variables, all the data are CPU local.
24 - If a softirq needs serialization, let it serialize itself
25 by its own spinlocks.
26 - Even if softirq is serialized, only local cpu is marked for
27 execution. Hence, we get something sort of weak cpu binding.
28 Though it is still not clear, will it result in better locality
29 or will not.
31 Examples:
32 - NET RX softirq. It is multithreaded and does not require
33 any global serialization.
34 - NET TX softirq. It kicks software netdevice queues, hence
35 it is logically serialized per device, but this serialization
36 is invisible to common code.
37 - Tasklets: serialized wrt itself.
40 #ifndef __ARCH_IRQ_STAT
41 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
42 EXPORT_SYMBOL(irq_stat);
43 #endif
45 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
47 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
50 * we cannot loop indefinitely here to avoid userspace starvation,
51 * but we also don't want to introduce a worst case 1/HZ latency
52 * to the pending events, so lets the scheduler to balance
53 * the softirq load for us.
55 static inline void wakeup_softirqd(void)
57 /* Interrupts are disabled: no need to stop preemption */
58 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
60 if (tsk && tsk->state != TASK_RUNNING)
61 wake_up_process(tsk);
65 * This one is for softirq.c-internal use,
66 * where hardirqs are disabled legitimately:
68 #ifdef CONFIG_TRACE_IRQFLAGS
69 static void __local_bh_disable(unsigned long ip)
71 unsigned long flags;
73 WARN_ON_ONCE(in_irq());
75 raw_local_irq_save(flags);
76 add_preempt_count(SOFTIRQ_OFFSET);
78 * Were softirqs turned off above:
80 if (softirq_count() == SOFTIRQ_OFFSET)
81 trace_softirqs_off(ip);
82 raw_local_irq_restore(flags);
84 #else /* !CONFIG_TRACE_IRQFLAGS */
85 static inline void __local_bh_disable(unsigned long ip)
87 add_preempt_count(SOFTIRQ_OFFSET);
88 barrier();
90 #endif /* CONFIG_TRACE_IRQFLAGS */
92 void local_bh_disable(void)
94 __local_bh_disable((unsigned long)__builtin_return_address(0));
97 EXPORT_SYMBOL(local_bh_disable);
99 void __local_bh_enable(void)
101 WARN_ON_ONCE(in_irq());
104 * softirqs should never be enabled by __local_bh_enable(),
105 * it always nests inside local_bh_enable() sections:
107 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
109 sub_preempt_count(SOFTIRQ_OFFSET);
111 EXPORT_SYMBOL_GPL(__local_bh_enable);
114 * Special-case - softirqs can safely be enabled in
115 * cond_resched_softirq(), or by __do_softirq(),
116 * without processing still-pending softirqs:
118 void _local_bh_enable(void)
120 WARN_ON_ONCE(in_irq());
121 WARN_ON_ONCE(!irqs_disabled());
123 if (softirq_count() == SOFTIRQ_OFFSET)
124 trace_softirqs_on((unsigned long)__builtin_return_address(0));
125 sub_preempt_count(SOFTIRQ_OFFSET);
128 EXPORT_SYMBOL(_local_bh_enable);
130 void local_bh_enable(void)
132 #ifdef CONFIG_TRACE_IRQFLAGS
133 unsigned long flags;
135 WARN_ON_ONCE(in_irq());
136 #endif
137 WARN_ON_ONCE(irqs_disabled());
139 #ifdef CONFIG_TRACE_IRQFLAGS
140 local_irq_save(flags);
141 #endif
143 * Are softirqs going to be turned on now:
145 if (softirq_count() == SOFTIRQ_OFFSET)
146 trace_softirqs_on((unsigned long)__builtin_return_address(0));
148 * Keep preemption disabled until we are done with
149 * softirq processing:
151 sub_preempt_count(SOFTIRQ_OFFSET - 1);
153 if (unlikely(!in_interrupt() && local_softirq_pending()))
154 do_softirq();
156 dec_preempt_count();
157 #ifdef CONFIG_TRACE_IRQFLAGS
158 local_irq_restore(flags);
159 #endif
160 preempt_check_resched();
162 EXPORT_SYMBOL(local_bh_enable);
164 void local_bh_enable_ip(unsigned long ip)
166 #ifdef CONFIG_TRACE_IRQFLAGS
167 unsigned long flags;
169 WARN_ON_ONCE(in_irq());
171 local_irq_save(flags);
172 #endif
174 * Are softirqs going to be turned on now:
176 if (softirq_count() == SOFTIRQ_OFFSET)
177 trace_softirqs_on(ip);
179 * Keep preemption disabled until we are done with
180 * softirq processing:
182 sub_preempt_count(SOFTIRQ_OFFSET - 1);
184 if (unlikely(!in_interrupt() && local_softirq_pending()))
185 do_softirq();
187 dec_preempt_count();
188 #ifdef CONFIG_TRACE_IRQFLAGS
189 local_irq_restore(flags);
190 #endif
191 preempt_check_resched();
193 EXPORT_SYMBOL(local_bh_enable_ip);
196 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
197 * and we fall back to softirqd after that.
199 * This number has been established via experimentation.
200 * The two things to balance is latency against fairness -
201 * we want to handle softirqs as soon as possible, but they
202 * should not be able to lock up the box.
204 #define MAX_SOFTIRQ_RESTART 10
206 asmlinkage void __do_softirq(void)
208 struct softirq_action *h;
209 __u32 pending;
210 int max_restart = MAX_SOFTIRQ_RESTART;
211 int cpu;
213 pending = local_softirq_pending();
214 account_system_vtime(current);
216 __local_bh_disable((unsigned long)__builtin_return_address(0));
217 trace_softirq_enter();
219 cpu = smp_processor_id();
220 restart:
221 /* Reset the pending bitmask before enabling irqs */
222 set_softirq_pending(0);
224 local_irq_enable();
226 h = softirq_vec;
228 do {
229 if (pending & 1) {
230 h->action(h);
231 rcu_bh_qsctr_inc(cpu);
233 h++;
234 pending >>= 1;
235 } while (pending);
237 local_irq_disable();
239 pending = local_softirq_pending();
240 if (pending && --max_restart)
241 goto restart;
243 if (pending)
244 wakeup_softirqd();
246 trace_softirq_exit();
248 account_system_vtime(current);
249 _local_bh_enable();
252 #ifndef __ARCH_HAS_DO_SOFTIRQ
254 asmlinkage void do_softirq(void)
256 __u32 pending;
257 unsigned long flags;
259 if (in_interrupt())
260 return;
262 local_irq_save(flags);
264 pending = local_softirq_pending();
266 if (pending)
267 __do_softirq();
269 local_irq_restore(flags);
272 EXPORT_SYMBOL(do_softirq);
274 #endif
276 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
277 # define invoke_softirq() __do_softirq()
278 #else
279 # define invoke_softirq() do_softirq()
280 #endif
283 * Exit an interrupt context. Process softirqs if needed and possible:
285 void irq_exit(void)
287 account_system_vtime(current);
288 trace_hardirq_exit();
289 sub_preempt_count(IRQ_EXIT_OFFSET);
290 if (!in_interrupt() && local_softirq_pending())
291 invoke_softirq();
292 preempt_enable_no_resched();
296 * This function must run with irqs disabled!
298 inline fastcall void raise_softirq_irqoff(unsigned int nr)
300 __raise_softirq_irqoff(nr);
303 * If we're in an interrupt or softirq, we're done
304 * (this also catches softirq-disabled code). We will
305 * actually run the softirq once we return from
306 * the irq or softirq.
308 * Otherwise we wake up ksoftirqd to make sure we
309 * schedule the softirq soon.
311 if (!in_interrupt())
312 wakeup_softirqd();
315 EXPORT_SYMBOL(raise_softirq_irqoff);
317 void fastcall raise_softirq(unsigned int nr)
319 unsigned long flags;
321 local_irq_save(flags);
322 raise_softirq_irqoff(nr);
323 local_irq_restore(flags);
326 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
328 softirq_vec[nr].data = data;
329 softirq_vec[nr].action = action;
332 /* Tasklets */
333 struct tasklet_head
335 struct tasklet_struct *list;
338 /* Some compilers disobey section attribute on statics when not
339 initialized -- RR */
340 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
341 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
343 void fastcall __tasklet_schedule(struct tasklet_struct *t)
345 unsigned long flags;
347 local_irq_save(flags);
348 t->next = __get_cpu_var(tasklet_vec).list;
349 __get_cpu_var(tasklet_vec).list = t;
350 raise_softirq_irqoff(TASKLET_SOFTIRQ);
351 local_irq_restore(flags);
354 EXPORT_SYMBOL(__tasklet_schedule);
356 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
358 unsigned long flags;
360 local_irq_save(flags);
361 t->next = __get_cpu_var(tasklet_hi_vec).list;
362 __get_cpu_var(tasklet_hi_vec).list = t;
363 raise_softirq_irqoff(HI_SOFTIRQ);
364 local_irq_restore(flags);
367 EXPORT_SYMBOL(__tasklet_hi_schedule);
369 static void tasklet_action(struct softirq_action *a)
371 struct tasklet_struct *list;
373 local_irq_disable();
374 list = __get_cpu_var(tasklet_vec).list;
375 __get_cpu_var(tasklet_vec).list = NULL;
376 local_irq_enable();
378 while (list) {
379 struct tasklet_struct *t = list;
381 list = list->next;
383 if (tasklet_trylock(t)) {
384 if (!atomic_read(&t->count)) {
385 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
386 BUG();
387 t->func(t->data);
388 tasklet_unlock(t);
389 continue;
391 tasklet_unlock(t);
394 local_irq_disable();
395 t->next = __get_cpu_var(tasklet_vec).list;
396 __get_cpu_var(tasklet_vec).list = t;
397 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
398 local_irq_enable();
402 static void tasklet_hi_action(struct softirq_action *a)
404 struct tasklet_struct *list;
406 local_irq_disable();
407 list = __get_cpu_var(tasklet_hi_vec).list;
408 __get_cpu_var(tasklet_hi_vec).list = NULL;
409 local_irq_enable();
411 while (list) {
412 struct tasklet_struct *t = list;
414 list = list->next;
416 if (tasklet_trylock(t)) {
417 if (!atomic_read(&t->count)) {
418 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
419 BUG();
420 t->func(t->data);
421 tasklet_unlock(t);
422 continue;
424 tasklet_unlock(t);
427 local_irq_disable();
428 t->next = __get_cpu_var(tasklet_hi_vec).list;
429 __get_cpu_var(tasklet_hi_vec).list = t;
430 __raise_softirq_irqoff(HI_SOFTIRQ);
431 local_irq_enable();
436 void tasklet_init(struct tasklet_struct *t,
437 void (*func)(unsigned long), unsigned long data)
439 t->next = NULL;
440 t->state = 0;
441 atomic_set(&t->count, 0);
442 t->func = func;
443 t->data = data;
446 EXPORT_SYMBOL(tasklet_init);
448 void tasklet_kill(struct tasklet_struct *t)
450 if (in_interrupt())
451 printk("Attempt to kill tasklet from interrupt\n");
453 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
455 yield();
456 while (test_bit(TASKLET_STATE_SCHED, &t->state));
458 tasklet_unlock_wait(t);
459 clear_bit(TASKLET_STATE_SCHED, &t->state);
462 EXPORT_SYMBOL(tasklet_kill);
464 void __init softirq_init(void)
466 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
467 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
470 static int ksoftirqd(void * __bind_cpu)
472 set_user_nice(current, 19);
473 current->flags |= PF_NOFREEZE;
475 set_current_state(TASK_INTERRUPTIBLE);
477 while (!kthread_should_stop()) {
478 preempt_disable();
479 if (!local_softirq_pending()) {
480 preempt_enable_no_resched();
481 schedule();
482 preempt_disable();
485 __set_current_state(TASK_RUNNING);
487 while (local_softirq_pending()) {
488 /* Preempt disable stops cpu going offline.
489 If already offline, we'll be on wrong CPU:
490 don't process */
491 if (cpu_is_offline((long)__bind_cpu))
492 goto wait_to_die;
493 do_softirq();
494 preempt_enable_no_resched();
495 cond_resched();
496 preempt_disable();
498 preempt_enable();
499 set_current_state(TASK_INTERRUPTIBLE);
501 __set_current_state(TASK_RUNNING);
502 return 0;
504 wait_to_die:
505 preempt_enable();
506 /* Wait for kthread_stop */
507 set_current_state(TASK_INTERRUPTIBLE);
508 while (!kthread_should_stop()) {
509 schedule();
510 set_current_state(TASK_INTERRUPTIBLE);
512 __set_current_state(TASK_RUNNING);
513 return 0;
516 #ifdef CONFIG_HOTPLUG_CPU
518 * tasklet_kill_immediate is called to remove a tasklet which can already be
519 * scheduled for execution on @cpu.
521 * Unlike tasklet_kill, this function removes the tasklet
522 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
524 * When this function is called, @cpu must be in the CPU_DEAD state.
526 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
528 struct tasklet_struct **i;
530 BUG_ON(cpu_online(cpu));
531 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
533 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
534 return;
536 /* CPU is dead, so no lock needed. */
537 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
538 if (*i == t) {
539 *i = t->next;
540 return;
543 BUG();
546 static void takeover_tasklets(unsigned int cpu)
548 struct tasklet_struct **i;
550 /* CPU is dead, so no lock needed. */
551 local_irq_disable();
553 /* Find end, append list for that CPU. */
554 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
555 *i = per_cpu(tasklet_vec, cpu).list;
556 per_cpu(tasklet_vec, cpu).list = NULL;
557 raise_softirq_irqoff(TASKLET_SOFTIRQ);
559 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
560 *i = per_cpu(tasklet_hi_vec, cpu).list;
561 per_cpu(tasklet_hi_vec, cpu).list = NULL;
562 raise_softirq_irqoff(HI_SOFTIRQ);
564 local_irq_enable();
566 #endif /* CONFIG_HOTPLUG_CPU */
568 static int __cpuinit cpu_callback(struct notifier_block *nfb,
569 unsigned long action,
570 void *hcpu)
572 int hotcpu = (unsigned long)hcpu;
573 struct task_struct *p;
575 switch (action) {
576 case CPU_UP_PREPARE:
577 BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
578 BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
579 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
580 if (IS_ERR(p)) {
581 printk("ksoftirqd for %i failed\n", hotcpu);
582 return NOTIFY_BAD;
584 kthread_bind(p, hotcpu);
585 per_cpu(ksoftirqd, hotcpu) = p;
586 break;
587 case CPU_ONLINE:
588 wake_up_process(per_cpu(ksoftirqd, hotcpu));
589 break;
590 #ifdef CONFIG_HOTPLUG_CPU
591 case CPU_UP_CANCELED:
592 if (!per_cpu(ksoftirqd, hotcpu))
593 break;
594 /* Unbind so it can run. Fall thru. */
595 kthread_bind(per_cpu(ksoftirqd, hotcpu),
596 any_online_cpu(cpu_online_map));
597 case CPU_DEAD:
598 p = per_cpu(ksoftirqd, hotcpu);
599 per_cpu(ksoftirqd, hotcpu) = NULL;
600 kthread_stop(p);
601 takeover_tasklets(hotcpu);
602 break;
603 #endif /* CONFIG_HOTPLUG_CPU */
605 return NOTIFY_OK;
608 static struct notifier_block __cpuinitdata cpu_nfb = {
609 .notifier_call = cpu_callback
612 __init int spawn_ksoftirqd(void)
614 void *cpu = (void *)(long)smp_processor_id();
615 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
617 BUG_ON(err == NOTIFY_BAD);
618 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
619 register_cpu_notifier(&cpu_nfb);
620 return 0;
623 #ifdef CONFIG_SMP
625 * Call a function on all processors
627 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
629 int ret = 0;
631 preempt_disable();
632 ret = smp_call_function(func, info, retry, wait);
633 local_irq_disable();
634 func(info);
635 local_irq_enable();
636 preempt_enable();
637 return ret;
639 EXPORT_SYMBOL(on_each_cpu);
640 #endif