kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/init.h>
  13 #include <linux/mm.h>
  14 #include <linux/notifier.h>
  15 #include <linux/percpu.h>
  16 #include <linux/cpu.h>
  17 #include <linux/kthread.h>
  18 #include <linux/rcupdate.h>
  19 #include <linux/smp.h>
  20
  21 #include <asm/irq.h>
  22 /*
  23    - No shared variables, all the data are CPU local.
  24    - If a softirq needs serialization, let it serialize itself
  25      by its own spinlocks.
  26    - Even if softirq is serialized, only local cpu is marked for
  27      execution. Hence, we get something sort of weak cpu binding.
  28      Though it is still not clear, will it result in better locality
  29      or will not.
  30
  31    Examples:
  32    - NET RX softirq. It is multithreaded and does not require
  33      any global serialization.
  34    - NET TX softirq. It kicks software netdevice queues, hence
  35      it is logically serialized per device, but this serialization
  36      is invisible to common code.
  37    - Tasklets: serialized wrt itself.
  38  */
  39
  40 #ifndef __ARCH_IRQ_STAT
  41 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  42 EXPORT_SYMBOL(irq_stat);
  43 #endif
  44
  45 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  46
  47 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  48
  49 /*
  50  * we cannot loop indefinitely here to avoid userspace starvation,
  51  * but we also don't want to introduce a worst case 1/HZ latency
  52  * to the pending events, so lets the scheduler to balance
  53  * the softirq load for us.
  54  */
  55 static inline void wakeup_softirqd(void)
  56 {
  57         /* Interrupts are disabled: no need to stop preemption */
  58         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  59
  60         if (tsk && tsk->state != TASK_RUNNING)
  61                 wake_up_process(tsk);
  62 }
  63
  64 static inline int softirqd_is_waken(void)
  65 {
  66         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  67
  68         return tsk && tsk->state == TASK_RUNNING;
  69 }
  70
  71 /*
  72  * This one is for softirq.c-internal use,
  73  * where hardirqs are disabled legitimately:
  74  */
  75 #ifdef CONFIG_TRACE_IRQFLAGS
  76 static void __local_bh_disable(unsigned long ip)
  77 {
  78         unsigned long flags;
  79
  80         WARN_ON_ONCE(in_irq());
  81
  82         raw_local_irq_save(flags);
  83         add_preempt_count(SOFTIRQ_OFFSET);
  84         /*
  85          * Were softirqs turned off above:
  86          */
  87         if (softirq_count() == SOFTIRQ_OFFSET)
  88                 trace_softirqs_off(ip);
  89         raw_local_irq_restore(flags);
  90 }
  91 #else /* !CONFIG_TRACE_IRQFLAGS */
  92 static inline void __local_bh_disable(unsigned long ip)
  93 {
  94         add_preempt_count(SOFTIRQ_OFFSET);
  95         barrier();
  96 }
  97 #endif /* CONFIG_TRACE_IRQFLAGS */
  98
  99 void local_bh_disable(void)
 100 {
 101         __local_bh_disable((unsigned long)__builtin_return_address(0));
 102 }
 103
 104 EXPORT_SYMBOL(local_bh_disable);
 105
 106 void __local_bh_enable(void)
 107 {
 108         WARN_ON_ONCE(in_irq());
 109
 110         /*
 111          * softirqs should never be enabled by __local_bh_enable(),
 112          * it always nests inside local_bh_enable() sections:
 113          */
 114         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 115
 116         sub_preempt_count(SOFTIRQ_OFFSET);
 117 }
 118 EXPORT_SYMBOL_GPL(__local_bh_enable);
 119
 120 /*
 121  * Special-case - softirqs can safely be enabled in
 122  * cond_resched_softirq(), or by __do_softirq(),
 123  * without processing still-pending softirqs:
 124  */
 125 void _local_bh_enable(void)
 126 {
 127         WARN_ON_ONCE(in_irq());
 128         WARN_ON_ONCE(!irqs_disabled());
 129
 130         if (softirq_count() == SOFTIRQ_OFFSET)
 131                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 132         sub_preempt_count(SOFTIRQ_OFFSET);
 133 }
 134
 135 EXPORT_SYMBOL(_local_bh_enable);
 136
 137 void local_bh_enable(void)
 138 {
 139 #ifdef CONFIG_TRACE_IRQFLAGS
 140         unsigned long flags;
 141
 142         WARN_ON_ONCE(in_irq());
 143 #endif
 144         WARN_ON_ONCE(irqs_disabled());
 145
 146 #ifdef CONFIG_TRACE_IRQFLAGS
 147         local_irq_save(flags);
 148 #endif
 149         /*
 150          * Are softirqs going to be turned on now:
 151          */
 152         if (softirq_count() == SOFTIRQ_OFFSET)
 153                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 154         /*
 155          * Keep preemption disabled until we are done with
 156          * softirq processing:
 157          */
 158         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 159
 160         if (unlikely(!in_interrupt() && local_softirq_pending()))
 161                 do_softirq();
 162
 163         dec_preempt_count();
 164 #ifdef CONFIG_TRACE_IRQFLAGS
 165         local_irq_restore(flags);
 166 #endif
 167         preempt_check_resched();
 168 }
 169 EXPORT_SYMBOL(local_bh_enable);
 170
 171 void local_bh_enable_ip(unsigned long ip)
 172 {
 173 #ifdef CONFIG_TRACE_IRQFLAGS
 174         unsigned long flags;
 175
 176         WARN_ON_ONCE(in_irq());
 177
 178         local_irq_save(flags);
 179 #endif
 180         /*
 181          * Are softirqs going to be turned on now:
 182          */
 183         if (softirq_count() == SOFTIRQ_OFFSET)
 184                 trace_softirqs_on(ip);
 185         /*
 186          * Keep preemption disabled until we are done with
 187          * softirq processing:
 188          */
 189         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 190
 191         if (unlikely(!in_interrupt() && local_softirq_pending()))
 192                 do_softirq();
 193
 194         dec_preempt_count();
 195 #ifdef CONFIG_TRACE_IRQFLAGS
 196         local_irq_restore(flags);
 197 #endif
 198         preempt_check_resched();
 199 }
 200 EXPORT_SYMBOL(local_bh_enable_ip);
 201
 202 /*
 203  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 204  * and we fall back to softirqd after that.
 205  *
 206  * This number has been established via experimentation.
 207  * The two things to balance is latency against fairness -
 208  * we want to handle softirqs as soon as possible, but they
 209  * should not be able to lock up the box.
 210  */
 211 #define MAX_SOFTIRQ_RESTART 10
 212
 213 static asmlinkage void __do_softirq2(void)
 214 {
 215         struct softirq_action *h;
 216         __u32 pending;
 217         int max_restart = MAX_SOFTIRQ_RESTART;
 218         int cpu;
 219
 220         pending = local_softirq_pending();
 221         account_system_vtime(current);
 222
 223         __local_bh_disable((unsigned long)__builtin_return_address(0));
 224         trace_softirq_enter();
 225
 226         cpu = smp_processor_id();
 227 restart:
 228         /* Reset the pending bitmask before enabling irqs */
 229         set_softirq_pending(0);
 230
 231         local_irq_enable();
 232
 233         h = softirq_vec;
 234
 235         do {
 236                 if (pending & 1) {
 237                         h->action(h);
 238                         rcu_bh_qsctr_inc(cpu);
 239                 }
 240                 h++;
 241                 pending >>= 1;
 242         } while (pending);
 243
 244         local_irq_disable();
 245
 246         pending = local_softirq_pending();
 247         if (pending && --max_restart)
 248                 goto restart;
 249
 250         if (pending)
 251                 wakeup_softirqd();
 252
 253         trace_softirq_exit();
 254
 255         account_system_vtime(current);
 256         _local_bh_enable();
 257 }
 258
 259 asmlinkage void __do_softirq(void)
 260 {
 261         if (!softirqd_is_waken())
 262                 __do_softirq2();
 263 }
 264
 265 #ifndef __ARCH_HAS_DO_SOFTIRQ
 266
 267 asmlinkage void do_softirq(void)
 268 {
 269         __u32 pending;
 270         unsigned long flags;
 271
 272         if (in_interrupt())
 273                 return;
 274
 275         local_irq_save(flags);
 276
 277         pending = local_softirq_pending();
 278
 279         if (pending)
 280                 __do_softirq();
 281
 282         local_irq_restore(flags);
 283 }
 284
 285 EXPORT_SYMBOL(do_softirq);
 286
 287 #endif
 288
 289 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 290 # define invoke_softirq()       __do_softirq()
 291 #else
 292 # define invoke_softirq()       do_softirq()
 293 #endif
 294
 295 /*
 296  * Exit an interrupt context. Process softirqs if needed and possible:
 297  */
 298 void irq_exit(void)
 299 {
 300         account_system_vtime(current);
 301         trace_hardirq_exit();
 302         sub_preempt_count(IRQ_EXIT_OFFSET);
 303         if (!in_interrupt() && local_softirq_pending())
 304                 invoke_softirq();
 305         preempt_enable_no_resched();
 306 }
 307
 308 /*
 309  * This function must run with irqs disabled!
 310  */
 311 inline fastcall void raise_softirq_irqoff(unsigned int nr)
 312 {
 313         __raise_softirq_irqoff(nr);
 314
 315         /*
 316          * If we're in an interrupt or softirq, we're done
 317          * (this also catches softirq-disabled code). We will
 318          * actually run the softirq once we return from
 319          * the irq or softirq.
 320          *
 321          * Otherwise we wake up ksoftirqd to make sure we
 322          * schedule the softirq soon.
 323          */
 324         if (!in_interrupt())
 325                 wakeup_softirqd();
 326 }
 327
 328 EXPORT_SYMBOL(raise_softirq_irqoff);
 329
 330 void fastcall raise_softirq(unsigned int nr)
 331 {
 332         unsigned long flags;
 333
 334         local_irq_save(flags);
 335         raise_softirq_irqoff(nr);
 336         local_irq_restore(flags);
 337 }
 338
 339 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 340 {
 341         softirq_vec[nr].data = data;
 342         softirq_vec[nr].action = action;
 343 }
 344
 345 /* Tasklets */
 346 struct tasklet_head
 347 {
 348         struct tasklet_struct *list;
 349 };
 350
 351 /* Some compilers disobey section attribute on statics when not
 352    initialized -- RR */
 353 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 354 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 355
 356 void fastcall __tasklet_schedule(struct tasklet_struct *t)
 357 {
 358         unsigned long flags;
 359
 360         local_irq_save(flags);
 361         t->next = __get_cpu_var(tasklet_vec).list;
 362         __get_cpu_var(tasklet_vec).list = t;
 363         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 364         local_irq_restore(flags);
 365 }
 366
 367 EXPORT_SYMBOL(__tasklet_schedule);
 368
 369 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
 370 {
 371         unsigned long flags;
 372
 373         local_irq_save(flags);
 374         t->next = __get_cpu_var(tasklet_hi_vec).list;
 375         __get_cpu_var(tasklet_hi_vec).list = t;
 376         raise_softirq_irqoff(HI_SOFTIRQ);
 377         local_irq_restore(flags);
 378 }
 379
 380 EXPORT_SYMBOL(__tasklet_hi_schedule);
 381
 382 static void tasklet_action(struct softirq_action *a)
 383 {
 384         struct tasklet_struct *list;
 385
 386         local_irq_disable();
 387         list = __get_cpu_var(tasklet_vec).list;
 388         __get_cpu_var(tasklet_vec).list = NULL;
 389         local_irq_enable();
 390
 391         while (list) {
 392                 struct tasklet_struct *t = list;
 393
 394                 list = list->next;
 395
 396                 if (tasklet_trylock(t)) {
 397                         if (!atomic_read(&t->count)) {
 398                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 399                                         BUG();
 400                                 t->func(t->data);
 401                                 tasklet_unlock(t);
 402                                 continue;
 403                         }
 404                         tasklet_unlock(t);
 405                 }
 406
 407                 local_irq_disable();
 408                 t->next = __get_cpu_var(tasklet_vec).list;
 409                 __get_cpu_var(tasklet_vec).list = t;
 410                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 411                 local_irq_enable();
 412         }
 413 }
 414
 415 static void tasklet_hi_action(struct softirq_action *a)
 416 {
 417         struct tasklet_struct *list;
 418
 419         local_irq_disable();
 420         list = __get_cpu_var(tasklet_hi_vec).list;
 421         __get_cpu_var(tasklet_hi_vec).list = NULL;
 422         local_irq_enable();
 423
 424         while (list) {
 425                 struct tasklet_struct *t = list;
 426
 427                 list = list->next;
 428
 429                 if (tasklet_trylock(t)) {
 430                         if (!atomic_read(&t->count)) {
 431                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 432                                         BUG();
 433                                 t->func(t->data);
 434                                 tasklet_unlock(t);
 435                                 continue;
 436                         }
 437                         tasklet_unlock(t);
 438                 }
 439
 440                 local_irq_disable();
 441                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 442                 __get_cpu_var(tasklet_hi_vec).list = t;
 443                 __raise_softirq_irqoff(HI_SOFTIRQ);
 444                 local_irq_enable();
 445         }
 446 }
 447
 448
 449 void tasklet_init(struct tasklet_struct *t,
 450                   void (*func)(unsigned long), unsigned long data)
 451 {
 452         t->next = NULL;
 453         t->state = 0;
 454         atomic_set(&t->count, 0);
 455         t->func = func;
 456         t->data = data;
 457 }
 458
 459 EXPORT_SYMBOL(tasklet_init);
 460
 461 void tasklet_kill(struct tasklet_struct *t)
 462 {
 463         if (in_interrupt())
 464                 printk("Attempt to kill tasklet from interrupt\n");
 465
 466         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 467                 do
 468                         yield();
 469                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 470         }
 471         tasklet_unlock_wait(t);
 472         clear_bit(TASKLET_STATE_SCHED, &t->state);
 473 }
 474
 475 EXPORT_SYMBOL(tasklet_kill);
 476
 477 void __init softirq_init(void)
 478 {
 479         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 480         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 481 }
 482
 483 static int ksoftirqd(void * __bind_cpu)
 484 {
 485         unsigned long flags;
 486
 487         set_user_nice(current, 19);
 488         current->flags |= PF_NOFREEZE;
 489
 490         set_current_state(TASK_INTERRUPTIBLE);
 491
 492         while (!kthread_should_stop()) {
 493                 preempt_disable();
 494                 if (!local_softirq_pending()) {
 495                         preempt_enable_no_resched();
 496                         schedule();
 497                         preempt_disable();
 498                 }
 499
 500                 __set_current_state(TASK_RUNNING);
 501
 502                 while (local_softirq_pending()) {
 503                         /* Preempt disable stops cpu going offline.
 504                            If already offline, we'll be on wrong CPU:
 505                            don't process */
 506                         if (cpu_is_offline((long)__bind_cpu))
 507                                 goto wait_to_die;
 508
 509                         local_irq_save(flags);
 510                         __do_softirq2();
 511                         local_irq_restore(flags);
 512
 513                         preempt_enable_no_resched();
 514                         cond_resched();
 515                         preempt_disable();
 516                 }
 517                 preempt_enable();
 518                 set_current_state(TASK_INTERRUPTIBLE);
 519         }
 520         __set_current_state(TASK_RUNNING);
 521         return 0;
 522
 523 wait_to_die:
 524         preempt_enable();
 525         /* Wait for kthread_stop */
 526         set_current_state(TASK_INTERRUPTIBLE);
 527         while (!kthread_should_stop()) {
 528                 schedule();
 529                 set_current_state(TASK_INTERRUPTIBLE);
 530         }
 531         __set_current_state(TASK_RUNNING);
 532         return 0;
 533 }
 534
 535 #ifdef CONFIG_HOTPLUG_CPU
 536 /*
 537  * tasklet_kill_immediate is called to remove a tasklet which can already be
 538  * scheduled for execution on @cpu.
 539  *
 540  * Unlike tasklet_kill, this function removes the tasklet
 541  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 542  *
 543  * When this function is called, @cpu must be in the CPU_DEAD state.
 544  */
 545 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 546 {
 547         struct tasklet_struct **i;
 548
 549         BUG_ON(cpu_online(cpu));
 550         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 551
 552         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 553                 return;
 554
 555         /* CPU is dead, so no lock needed. */
 556         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
 557                 if (*i == t) {
 558                         *i = t->next;
 559                         return;
 560                 }
 561         }
 562         BUG();
 563 }
 564
 565 static void takeover_tasklets(unsigned int cpu)
 566 {
 567         struct tasklet_struct **i;
 568
 569         /* CPU is dead, so no lock needed. */
 570         local_irq_disable();
 571
 572         /* Find end, append list for that CPU. */
 573         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
 574         *i = per_cpu(tasklet_vec, cpu).list;
 575         per_cpu(tasklet_vec, cpu).list = NULL;
 576         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 577
 578         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
 579         *i = per_cpu(tasklet_hi_vec, cpu).list;
 580         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 581         raise_softirq_irqoff(HI_SOFTIRQ);
 582
 583         local_irq_enable();
 584 }
 585 #endif /* CONFIG_HOTPLUG_CPU */
 586
 587 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 588                                   unsigned long action,
 589                                   void *hcpu)
 590 {
 591         int hotcpu = (unsigned long)hcpu;
 592         struct task_struct *p;
 593
 594         switch (action) {
 595         case CPU_UP_PREPARE:
 596                 BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
 597                 BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
 598                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 599                 if (IS_ERR(p)) {
 600                         printk("ksoftirqd for %i failed\n", hotcpu);
 601                         return NOTIFY_BAD;
 602                 }
 603                 kthread_bind(p, hotcpu);
 604                 per_cpu(ksoftirqd, hotcpu) = p;
 605                 break;
 606         case CPU_ONLINE:
 607                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 608                 break;
 609 #ifdef CONFIG_HOTPLUG_CPU
 610         case CPU_UP_CANCELED:
 611                 if (!per_cpu(ksoftirqd, hotcpu))
 612                         break;
 613                 /* Unbind so it can run.  Fall thru. */
 614                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 615                              any_online_cpu(cpu_online_map));
 616         case CPU_DEAD:
 617                 p = per_cpu(ksoftirqd, hotcpu);
 618                 per_cpu(ksoftirqd, hotcpu) = NULL;
 619                 kthread_stop(p);
 620                 takeover_tasklets(hotcpu);
 621                 break;
 622 #endif /* CONFIG_HOTPLUG_CPU */
 623         }
 624         return NOTIFY_OK;
 625 }
 626
 627 static struct notifier_block __cpuinitdata cpu_nfb = {
 628         .notifier_call = cpu_callback
 629 };
 630
 631 __init int spawn_ksoftirqd(void)
 632 {
 633         void *cpu = (void *)(long)smp_processor_id();
 634         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 635
 636         BUG_ON(err == NOTIFY_BAD);
 637         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 638         register_cpu_notifier(&cpu_nfb);
 639         return 0;
 640 }
 641
 642 #ifdef CONFIG_SMP
 643 /*
 644  * Call a function on all processors
 645  */
 646 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
 647 {
 648         int ret = 0;
 649
 650         preempt_disable();
 651         ret = smp_call_function(func, info, retry, wait);
 652         local_irq_disable();
 653         func(info);
 654         local_irq_enable();
 655         preempt_enable();
 656         return ret;
 657 }
 658 EXPORT_SYMBOL(on_each_cpu);
 659 #endif