kernel/stop_machine.c

   1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
   2  * GPL v2 and any later version.
   3  */
   4 #include <linux/cpu.h>
   5 #include <linux/err.h>
   6 #include <linux/kthread.h>
   7 #include <linux/module.h>
   8 #include <linux/sched.h>
   9 #include <linux/stop_machine.h>
  10 #include <linux/syscalls.h>
  11 #include <linux/interrupt.h>
  12
  13 #include <asm/atomic.h>
  14 #include <asm/uaccess.h>
  15
  16 /* This controls the threads on each CPU. */
  17 enum stopmachine_state {
  18         /* Dummy starting state for thread. */
  19         STOPMACHINE_NONE,
  20         /* Awaiting everyone to be scheduled. */
  21         STOPMACHINE_PREPARE,
  22         /* Disable interrupts. */
  23         STOPMACHINE_DISABLE_IRQ,
  24         /* Run the function */
  25         STOPMACHINE_RUN,
  26         /* Exit */
  27         STOPMACHINE_EXIT,
  28 };
  29 static enum stopmachine_state state;
  30
  31 struct stop_machine_data {
  32         int (*fn)(void *);
  33         void *data;
  34         int fnret;
  35 };
  36
  37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
  38 static unsigned int num_threads;
  39 static atomic_t thread_ack;
  40 static struct completion finished;
  41 static DEFINE_MUTEX(lock);
  42
  43 static void set_state(enum stopmachine_state newstate)
  44 {
  45         /* Reset ack counter. */
  46         atomic_set(&thread_ack, num_threads);
  47         smp_wmb();
  48         state = newstate;
  49 }
  50
  51 /* Last one to ack a state moves to the next state. */
  52 static void ack_state(void)
  53 {
  54         if (atomic_dec_and_test(&thread_ack)) {
  55                 /* If we're the last one to ack the EXIT, we're finished. */
  56                 if (state == STOPMACHINE_EXIT)
  57                         complete(&finished);
  58                 else
  59                         set_state(state + 1);
  60         }
  61 }
  62
  63 /* This is the actual thread which stops the CPU.  It exits by itself rather
  64  * than waiting for kthread_stop(), because it's easier for hotplug CPU. */
  65 static int stop_cpu(struct stop_machine_data *smdata)
  66 {
  67         enum stopmachine_state curstate = STOPMACHINE_NONE;
  68         int uninitialized_var(ret);
  69
  70         /* Simple state machine */
  71         do {
  72                 /* Chill out and ensure we re-read stopmachine_state. */
  73                 cpu_relax();
  74                 if (state != curstate) {
  75                         curstate = state;
  76                         switch (curstate) {
  77                         case STOPMACHINE_DISABLE_IRQ:
  78                                 local_irq_disable();
  79                                 hard_irq_disable();
  80                                 break;
  81                         case STOPMACHINE_RUN:
  82                                 /* |= allows error detection if functions on
  83                                  * multiple CPUs. */
  84                                 smdata->fnret |= smdata->fn(smdata->data);
  85                                 break;
  86                         default:
  87                                 break;
  88                         }
  89                         ack_state();
  90                 }
  91         } while (curstate != STOPMACHINE_EXIT);
  92
  93         local_irq_enable();
  94         do_exit(0);
  95 }
  96
  97 /* Callback for CPUs which aren't supposed to do anything. */
  98 static int chill(void *unused)
  99 {
 100         return 0;
 101 }
 102
 103 int __stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
 104 {
 105         int i, err;
 106         struct stop_machine_data active, idle;
 107         struct task_struct **threads;
 108
 109         active.fn = fn;
 110         active.data = data;
 111         active.fnret = 0;
 112         idle.fn = chill;
 113         idle.data = NULL;
 114
 115         /* If they don't care which cpu fn runs on, just pick one. */
 116         if (cpu == NR_CPUS)
 117                 cpu = any_online_cpu(cpu_online_map);
 118
 119         /* This could be too big for stack on large machines. */
 120         threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL);
 121         if (!threads)
 122                 return -ENOMEM;
 123
 124         /* Set up initial state. */
 125         mutex_lock(&lock);
 126         init_completion(&finished);
 127         num_threads = num_online_cpus();
 128         set_state(STOPMACHINE_PREPARE);
 129
 130         for_each_online_cpu(i) {
 131                 struct stop_machine_data *smdata;
 132                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 133
 134                 if (cpu == ALL_CPUS || i == cpu)
 135                         smdata = &active;
 136                 else
 137                         smdata = &idle;
 138
 139                 threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u",
 140                                             i);
 141                 if (IS_ERR(threads[i])) {
 142                         err = PTR_ERR(threads[i]);
 143                         threads[i] = NULL;
 144                         goto kill_threads;
 145                 }
 146
 147                 /* Place it onto correct cpu. */
 148                 kthread_bind(threads[i], i);
 149
 150                 /* Make it highest prio. */
 151                 if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, &param))
 152                         BUG();
 153         }
 154
 155         /* We've created all the threads.  Wake them all: hold this CPU so one
 156          * doesn't hit this CPU until we're ready. */
 157         cpu = get_cpu();
 158         for_each_online_cpu(i)
 159                 wake_up_process(threads[i]);
 160
 161         /* This will release the thread on our CPU. */
 162         put_cpu();
 163         wait_for_completion(&finished);
 164         mutex_unlock(&lock);
 165
 166         kfree(threads);
 167
 168         return active.fnret;
 169
 170 kill_threads:
 171         for_each_online_cpu(i)
 172                 if (threads[i])
 173                         kthread_stop(threads[i]);
 174         mutex_unlock(&lock);
 175
 176         kfree(threads);
 177         return err;
 178 }
 179
 180 int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
 181 {
 182         int ret;
 183
 184         /* No CPUs can come up or down during this. */
 185         get_online_cpus();
 186         ret = __stop_machine_run(fn, data, cpu);
 187         put_online_cpus();
 188
 189         return ret;
 190 }
 191 EXPORT_SYMBOL_GPL(stop_machine_run);