kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/kthread.h>
  18 #include <linux/rcupdate.h>
  19 #include <linux/smp.h>
  20 #include <linux/tick.h>
  21
  22 #include <asm/irq.h>
  23 /*
  24    - No shared variables, all the data are CPU local.
  25    - If a softirq needs serialization, let it serialize itself
  26      by its own spinlocks.
  27    - Even if softirq is serialized, only local cpu is marked for
  28      execution. Hence, we get something sort of weak cpu binding.
  29      Though it is still not clear, will it result in better locality
  30      or will not.
  31
  32    Examples:
  33    - NET RX softirq. It is multithreaded and does not require
  34      any global serialization.
  35    - NET TX softirq. It kicks software netdevice queues, hence
  36      it is logically serialized per device, but this serialization
  37      is invisible to common code.
  38    - Tasklets: serialized wrt itself.
  39  */
  40
  41 #ifndef __ARCH_IRQ_STAT
  42 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  43 EXPORT_SYMBOL(irq_stat);
  44 #endif
  45
  46 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  47
  48 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  49
  50 /*
  51  * we cannot loop indefinitely here to avoid userspace starvation,
  52  * but we also don't want to introduce a worst case 1/HZ latency
  53  * to the pending events, so lets the scheduler to balance
  54  * the softirq load for us.
  55  */
  56 static inline void wakeup_softirqd(void)
  57 {
  58         /* Interrupts are disabled: no need to stop preemption */
  59         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  60
  61         if (tsk && tsk->state != TASK_RUNNING)
  62                 wake_up_process(tsk);
  63 }
  64
  65 /*
  66  * This one is for softirq.c-internal use,
  67  * where hardirqs are disabled legitimately:
  68  */
  69 #ifdef CONFIG_TRACE_IRQFLAGS
  70 static void __local_bh_disable(unsigned long ip)
  71 {
  72         unsigned long flags;
  73
  74         WARN_ON_ONCE(in_irq());
  75
  76         raw_local_irq_save(flags);
  77         add_preempt_count(SOFTIRQ_OFFSET);
  78         /*
  79          * Were softirqs turned off above:
  80          */
  81         if (softirq_count() == SOFTIRQ_OFFSET)
  82                 trace_softirqs_off(ip);
  83         raw_local_irq_restore(flags);
  84 }
  85 #else /* !CONFIG_TRACE_IRQFLAGS */
  86 static inline void __local_bh_disable(unsigned long ip)
  87 {
  88         add_preempt_count(SOFTIRQ_OFFSET);
  89         barrier();
  90 }
  91 #endif /* CONFIG_TRACE_IRQFLAGS */
  92
  93 void local_bh_disable(void)
  94 {
  95         __local_bh_disable((unsigned long)__builtin_return_address(0));
  96 }
  97
  98 EXPORT_SYMBOL(local_bh_disable);
  99
 100 void __local_bh_enable(void)
 101 {
 102         WARN_ON_ONCE(in_irq());
 103
 104         /*
 105          * softirqs should never be enabled by __local_bh_enable(),
 106          * it always nests inside local_bh_enable() sections:
 107          */
 108         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 109
 110         sub_preempt_count(SOFTIRQ_OFFSET);
 111 }
 112 EXPORT_SYMBOL_GPL(__local_bh_enable);
 113
 114 /*
 115  * Special-case - softirqs can safely be enabled in
 116  * cond_resched_softirq(), or by __do_softirq(),
 117  * without processing still-pending softirqs:
 118  */
 119 void _local_bh_enable(void)
 120 {
 121         WARN_ON_ONCE(in_irq());
 122         WARN_ON_ONCE(!irqs_disabled());
 123
 124         if (softirq_count() == SOFTIRQ_OFFSET)
 125                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 126         sub_preempt_count(SOFTIRQ_OFFSET);
 127 }
 128
 129 EXPORT_SYMBOL(_local_bh_enable);
 130
 131 void local_bh_enable(void)
 132 {
 133 #ifdef CONFIG_TRACE_IRQFLAGS
 134         unsigned long flags;
 135
 136         WARN_ON_ONCE(in_irq());
 137 #endif
 138         WARN_ON_ONCE(irqs_disabled());
 139
 140 #ifdef CONFIG_TRACE_IRQFLAGS
 141         local_irq_save(flags);
 142 #endif
 143         /*
 144          * Are softirqs going to be turned on now:
 145          */
 146         if (softirq_count() == SOFTIRQ_OFFSET)
 147                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 148         /*
 149          * Keep preemption disabled until we are done with
 150          * softirq processing:
 151          */
 152         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 153
 154         if (unlikely(!in_interrupt() && local_softirq_pending()))
 155                 do_softirq();
 156
 157         dec_preempt_count();
 158 #ifdef CONFIG_TRACE_IRQFLAGS
 159         local_irq_restore(flags);
 160 #endif
 161         preempt_check_resched();
 162 }
 163 EXPORT_SYMBOL(local_bh_enable);
 164
 165 void local_bh_enable_ip(unsigned long ip)
 166 {
 167 #ifdef CONFIG_TRACE_IRQFLAGS
 168         unsigned long flags;
 169
 170         WARN_ON_ONCE(in_irq());
 171
 172         local_irq_save(flags);
 173 #endif
 174         /*
 175          * Are softirqs going to be turned on now:
 176          */
 177         if (softirq_count() == SOFTIRQ_OFFSET)
 178                 trace_softirqs_on(ip);
 179         /*
 180          * Keep preemption disabled until we are done with
 181          * softirq processing:
 182          */
 183         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 184
 185         if (unlikely(!in_interrupt() && local_softirq_pending()))
 186                 do_softirq();
 187
 188         dec_preempt_count();
 189 #ifdef CONFIG_TRACE_IRQFLAGS
 190         local_irq_restore(flags);
 191 #endif
 192         preempt_check_resched();
 193 }
 194 EXPORT_SYMBOL(local_bh_enable_ip);
 195
 196 /*
 197  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 198  * and we fall back to softirqd after that.
 199  *
 200  * This number has been established via experimentation.
 201  * The two things to balance is latency against fairness -
 202  * we want to handle softirqs as soon as possible, but they
 203  * should not be able to lock up the box.
 204  */
 205 #define MAX_SOFTIRQ_RESTART 10
 206
 207 asmlinkage void __do_softirq(void)
 208 {
 209         struct softirq_action *h;
 210         __u32 pending;
 211         int max_restart = MAX_SOFTIRQ_RESTART;
 212         int cpu;
 213
 214         pending = local_softirq_pending();
 215         account_system_vtime(current);
 216
 217         __local_bh_disable((unsigned long)__builtin_return_address(0));
 218         trace_softirq_enter();
 219
 220         cpu = smp_processor_id();
 221 restart:
 222         /* Reset the pending bitmask before enabling irqs */
 223         set_softirq_pending(0);
 224
 225         local_irq_enable();
 226
 227         h = softirq_vec;
 228
 229         do {
 230                 if (pending & 1) {
 231                         h->action(h);
 232                         rcu_bh_qsctr_inc(cpu);
 233                 }
 234                 h++;
 235                 pending >>= 1;
 236         } while (pending);
 237
 238         local_irq_disable();
 239
 240         pending = local_softirq_pending();
 241         if (pending && --max_restart)
 242                 goto restart;
 243
 244         if (pending)
 245                 wakeup_softirqd();
 246
 247         trace_softirq_exit();
 248
 249         account_system_vtime(current);
 250         _local_bh_enable();
 251 }
 252
 253 #ifndef __ARCH_HAS_DO_SOFTIRQ
 254
 255 asmlinkage void do_softirq(void)
 256 {
 257         __u32 pending;
 258         unsigned long flags;
 259
 260         if (in_interrupt())
 261                 return;
 262
 263         local_irq_save(flags);
 264
 265         pending = local_softirq_pending();
 266
 267         if (pending)
 268                 __do_softirq();
 269
 270         local_irq_restore(flags);
 271 }
 272
 273 EXPORT_SYMBOL(do_softirq);
 274
 275 #endif
 276
 277 /*
 278  * Enter an interrupt context.
 279  */
 280 void irq_enter(void)
 281 {
 282         __irq_enter();
 283 #ifdef CONFIG_NO_HZ
 284         if (idle_cpu(smp_processor_id()))
 285                 tick_nohz_update_jiffies();
 286 #endif
 287 }
 288
 289 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 290 # define invoke_softirq()       __do_softirq()
 291 #else
 292 # define invoke_softirq()       do_softirq()
 293 #endif
 294
 295 /*
 296  * Exit an interrupt context. Process softirqs if needed and possible:
 297  */
 298 void irq_exit(void)
 299 {
 300         account_system_vtime(current);
 301         trace_hardirq_exit();
 302         sub_preempt_count(IRQ_EXIT_OFFSET);
 303         if (!in_interrupt() && local_softirq_pending())
 304                 invoke_softirq();
 305
 306 #ifdef CONFIG_NO_HZ
 307         /* Make sure that timer wheel updates are propagated */
 308         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
 309                 tick_nohz_stop_sched_tick();
 310 #endif
 311         preempt_enable_no_resched();
 312 }
 313
 314 /*
 315  * This function must run with irqs disabled!
 316  */
 317 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 318 {
 319         __raise_softirq_irqoff(nr);
 320
 321         /*
 322          * If we're in an interrupt or softirq, we're done
 323          * (this also catches softirq-disabled code). We will
 324          * actually run the softirq once we return from
 325          * the irq or softirq.
 326          *
 327          * Otherwise we wake up ksoftirqd to make sure we
 328          * schedule the softirq soon.
 329          */
 330         if (!in_interrupt())
 331                 wakeup_softirqd();
 332 }
 333
 334 EXPORT_SYMBOL(raise_softirq_irqoff);
 335
 336 void fastcall raise_softirq(unsigned int nr)
 337 {
 338         unsigned long flags;
 339
 340         local_irq_save(flags);
 341         raise_softirq_irqoff(nr);
 342         local_irq_restore(flags);
 343 }
 344
 345 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 346 {
 347         softirq_vec[nr].data = data;
 348         softirq_vec[nr].action = action;
 349 }
 350
 351 /* Tasklets */
 352 struct tasklet_head
 353 {
 354         struct tasklet_struct *list;
 355 };
 356
 357 /* Some compilers disobey section attribute on statics when not
 358    initialized -- RR */
 359 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 360 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 361
 362 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 363 {
 364         unsigned long flags;
 365
 366         local_irq_save(flags);
 367         t->next = __get_cpu_var(tasklet_vec).list;
 368         __get_cpu_var(tasklet_vec).list = t;
 369         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 370         local_irq_restore(flags);
 371 }
 372
 373 EXPORT_SYMBOL(__tasklet_schedule);
 374
 375 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 376 {
 377         unsigned long flags;
 378
 379         local_irq_save(flags);
 380         t->next = __get_cpu_var(tasklet_hi_vec).list;
 381         __get_cpu_var(tasklet_hi_vec).list = t;
 382         raise_softirq_irqoff(HI_SOFTIRQ);
 383         local_irq_restore(flags);
 384 }
 385
 386 EXPORT_SYMBOL(__tasklet_hi_schedule);
 387
 388 static void tasklet_action(struct softirq_action *a)
 389 {
 390         struct tasklet_struct *list;
 391
 392         local_irq_disable();
 393         list = __get_cpu_var(tasklet_vec).list;
 394         __get_cpu_var(tasklet_vec).list = NULL;
 395         local_irq_enable();
 396
 397         while (list) {
 398                 struct tasklet_struct *t = list;
 399
 400                 list = list->next;
 401
 402                 if (tasklet_trylock(t)) {
 403                         if (!atomic_read(&t->count)) {
 404                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 405                                         BUG();
 406                                 t->func(t->data);
 407                                 tasklet_unlock(t);
 408                                 continue;
 409                         }
 410                         tasklet_unlock(t);
 411                 }
 412
 413                 local_irq_disable();
 414                 t->next = __get_cpu_var(tasklet_vec).list;
 415                 __get_cpu_var(tasklet_vec).list = t;
 416                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 417                 local_irq_enable();
 418         }
 419 }
 420
 421 static void tasklet_hi_action(struct softirq_action *a)
 422 {
 423         struct tasklet_struct *list;
 424
 425         local_irq_disable();
 426         list = __get_cpu_var(tasklet_hi_vec).list;
 427         __get_cpu_var(tasklet_hi_vec).list = NULL;
 428         local_irq_enable();
 429
 430         while (list) {
 431                 struct tasklet_struct *t = list;
 432
 433                 list = list->next;
 434
 435                 if (tasklet_trylock(t)) {
 436                         if (!atomic_read(&t->count)) {
 437                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 438                                         BUG();
 439                                 t->func(t->data);
 440                                 tasklet_unlock(t);
 441                                 continue;
 442                         }
 443                         tasklet_unlock(t);
 444                 }
 445
 446                 local_irq_disable();
 447                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 448                 __get_cpu_var(tasklet_hi_vec).list = t;
 449                 __raise_softirq_irqoff(HI_SOFTIRQ);
 450                 local_irq_enable();
 451         }
 452 }
 453
 454
 455 void tasklet_init(struct tasklet_struct *t,
 456                   void (*func)(unsigned long), unsigned long data)
 457 {
 458         t->next = NULL;
 459         t->state = 0;
 460         atomic_set(&t->count, 0);
 461         t->func = func;
 462         t->data = data;
 463 }
 464
 465 EXPORT_SYMBOL(tasklet_init);
 466
 467 void tasklet_kill(struct tasklet_struct *t)
 468 {
 469         if (in_interrupt())
 470                 printk("Attempt to kill tasklet from interrupt\n");
 471
 472         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 473                 do
 474                         yield();
 475                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 476         }
 477         tasklet_unlock_wait(t);
 478         clear_bit(TASKLET_STATE_SCHED, &t->state);
 479 }
 480
 481 EXPORT_SYMBOL(tasklet_kill);
 482
 483 void __init softirq_init(void)
 484 {
 485         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 486         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 487 }
 488
 489 static int ksoftirqd(void * __bind_cpu)
 490 {
 491         current->flags |= PF_NOFREEZE;
 492
 493         set_current_state(TASK_INTERRUPTIBLE);
 494
 495         while (!kthread_should_stop()) {
 496                 preempt_disable();
 497                 if (!local_softirq_pending()) {
 498                         preempt_enable_no_resched();
 499                         schedule();
 500                         preempt_disable();
 501                 }
 502
 503                 __set_current_state(TASK_RUNNING);
 504
 505                 while (local_softirq_pending()) {
 506                         /* Preempt disable stops cpu going offline.
 507                            If already offline, we'll be on wrong CPU:
 508                            don't process */
 509                         if (cpu_is_offline((long)__bind_cpu))
 510                                 goto wait_to_die;
 511                         do_softirq();
 512                         preempt_enable_no_resched();
 513                         cond_resched();
 514                         preempt_disable();
 515                 }
 516                 preempt_enable();
 517                 set_current_state(TASK_INTERRUPTIBLE);
 518         }
 519         __set_current_state(TASK_RUNNING);
 520         return 0;
 521
 522 wait_to_die:
 523         preempt_enable();
 524         /* Wait for kthread_stop */
 525         set_current_state(TASK_INTERRUPTIBLE);
 526         while (!kthread_should_stop()) {
 527                 schedule();
 528                 set_current_state(TASK_INTERRUPTIBLE);
 529         }
 530         __set_current_state(TASK_RUNNING);
 531         return 0;
 532 }
 533
 534 #ifdef CONFIG_HOTPLUG_CPU
 535 /*
 536  * tasklet_kill_immediate is called to remove a tasklet which can already be
 537  * scheduled for execution on @cpu.
 538  *
 539  * Unlike tasklet_kill, this function removes the tasklet
 540  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 541  *
 542  * When this function is called, @cpu must be in the CPU_DEAD state.
 543  */
 544 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 545 {
 546         struct tasklet_struct **i;
 547
 548         BUG_ON(cpu_online(cpu));
 549         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 550
 551         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 552                 return;
 553
 554         /* CPU is dead, so no lock needed. */
 555         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 556                 if (*i == t) {
 557                         *i = t->next;
 558                         return;
 559                 }
 560         }
 561         BUG();
 562 }
 563
 564 static void takeover_tasklets(unsigned int cpu)
 565 {
 566         struct tasklet_struct **i;
 567
 568         /* CPU is dead, so no lock needed. */
 569         local_irq_disable();
 570
 571         /* Find end, append list for that CPU. */
 572         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 573         *i = per_cpu(tasklet_vec, cpu).list;
 574         per_cpu(tasklet_vec, cpu).list = NULL;
 575         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 576
 577         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 578         *i = per_cpu(tasklet_hi_vec, cpu).list;
 579         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 580         raise_softirq_irqoff(HI_SOFTIRQ);
 581
 582         local_irq_enable();
 583 }
 584 #endif /* CONFIG_HOTPLUG_CPU */
 585
 586 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 587                                   unsigned long action,
 588                                   void *hcpu)
 589 {
 590         int hotcpu = (unsigned long)hcpu;
 591         struct task_struct *p;
 592
 593         switch (action) {
 594         case CPU_UP_PREPARE:
 595         case CPU_UP_PREPARE_FROZEN:
 596                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 597                 if (IS_ERR(p)) {
 598                         printk("ksoftirqd for %i failed\n", hotcpu);
 599                         return NOTIFY_BAD;
 600                 }
 601                 kthread_bind(p, hotcpu);
 602                 per_cpu(ksoftirqd, hotcpu) = p;
 603                 break;
 604         case CPU_ONLINE:
 605         case CPU_ONLINE_FROZEN:
 606                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 607                 break;
 608 #ifdef CONFIG_HOTPLUG_CPU
 609         case CPU_UP_CANCELED:
 610         case CPU_UP_CANCELED_FROZEN:
 611                 if (!per_cpu(ksoftirqd, hotcpu))
 612                         break;
 613                 /* Unbind so it can run.  Fall thru. */
 614                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 615                              any_online_cpu(cpu_online_map));
 616         case CPU_DEAD:
 617         case CPU_DEAD_FROZEN: {
 618                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 619
 620                 p = per_cpu(ksoftirqd, hotcpu);
 621                 per_cpu(ksoftirqd, hotcpu) = NULL;
 622                 sched_setscheduler(p, SCHED_FIFO, &param);
 623                 kthread_stop(p);
 624                 takeover_tasklets(hotcpu);
 625                 break;
 626         }
 627 #endif /* CONFIG_HOTPLUG_CPU */
 628         }
 629         return NOTIFY_OK;
 630 }
 631
 632 static struct notifier_block __cpuinitdata cpu_nfb = {
 633         .notifier_call = cpu_callback
 634 };
 635
 636 __init int spawn_ksoftirqd(void)
 637 {
 638         void *cpu = (void *)(long)smp_processor_id();
 639         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 640
 641         BUG_ON(err == NOTIFY_BAD);
 642         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 643         register_cpu_notifier(&cpu_nfb);
 644         return 0;
 645 }
 646
 647 #ifdef CONFIG_SMP
 648 /*
 649  * Call a function on all processors
 650  */
 651 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
 652 {
 653         int ret = 0;
 654
 655         preempt_disable();
 656         ret = smp_call_function(func, info, retry, wait);
 657         local_irq_disable();
 658         func(info);
 659         local_irq_enable();
 660         preempt_enable();
 661         return ret;
 662 }
 663 EXPORT_SYMBOL(on_each_cpu);
 664 #endif