kernel/stop_machine.c

   1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
   2  * GPL v2 and any later version.
   3  */
   4 #include <linux/cpu.h>
   5 #include <linux/err.h>
   6 #include <linux/kthread.h>
   7 #include <linux/module.h>
   8 #include <linux/sched.h>
   9 #include <linux/stop_machine.h>
  10 #include <linux/syscalls.h>
  11 #include <linux/interrupt.h>
  12
  13 #include <asm/atomic.h>
  14 #include <asm/uaccess.h>
  15
  16 /* This controls the threads on each CPU. */
  17 enum stopmachine_state {
  18         /* Dummy starting state for thread. */
  19         STOPMACHINE_NONE,
  20         /* Awaiting everyone to be scheduled. */
  21         STOPMACHINE_PREPARE,
  22         /* Disable interrupts. */
  23         STOPMACHINE_DISABLE_IRQ,
  24         /* Run the function */
  25         STOPMACHINE_RUN,
  26         /* Exit */
  27         STOPMACHINE_EXIT,
  28 };
  29 static enum stopmachine_state state;
  30
  31 struct stop_machine_data {
  32         int (*fn)(void *);
  33         void *data;
  34         int fnret;
  35 };
  36
  37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
  38 static unsigned int num_threads;
  39 static atomic_t thread_ack;
  40 static DEFINE_MUTEX(lock);
  41 /* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */
  42 static DEFINE_MUTEX(setup_lock);
  43 /* Users of stop_machine. */
  44 static int refcount;
  45 static struct workqueue_struct *stop_machine_wq;
  46 static struct stop_machine_data active, idle;
  47 static const struct cpumask *active_cpus;
  48 static void __percpu *stop_machine_work;
  49
  50 static void set_state(enum stopmachine_state newstate)
  51 {
  52         /* Reset ack counter. */
  53         atomic_set(&thread_ack, num_threads);
  54         smp_wmb();
  55         state = newstate;
  56 }
  57
  58 /* Last one to ack a state moves to the next state. */
  59 static void ack_state(void)
  60 {
  61         if (atomic_dec_and_test(&thread_ack))
  62                 set_state(state + 1);
  63 }
  64
  65 /* This is the actual function which stops the CPU. It runs
  66  * in the context of a dedicated stopmachine workqueue. */
  67 static void stop_cpu(struct work_struct *unused)
  68 {
  69         enum stopmachine_state curstate = STOPMACHINE_NONE;
  70         struct stop_machine_data *smdata = &idle;
  71         int cpu = smp_processor_id();
  72         int err;
  73
  74         if (!active_cpus) {
  75                 if (cpu == cpumask_first(cpu_online_mask))
  76                         smdata = &active;
  77         } else {
  78                 if (cpumask_test_cpu(cpu, active_cpus))
  79                         smdata = &active;
  80         }
  81         /* Simple state machine */
  82         do {
  83                 /* Chill out and ensure we re-read stopmachine_state. */
  84                 cpu_relax();
  85                 if (state != curstate) {
  86                         curstate = state;
  87                         switch (curstate) {
  88                         case STOPMACHINE_DISABLE_IRQ:
  89                                 local_irq_disable();
  90                                 hard_irq_disable();
  91                                 break;
  92                         case STOPMACHINE_RUN:
  93                                 /* On multiple CPUs only a single error code
  94                                  * is needed to tell that something failed. */
  95                                 err = smdata->fn(smdata->data);
  96                                 if (err)
  97                                         smdata->fnret = err;
  98                                 break;
  99                         default:
 100                                 break;
 101                         }
 102                         ack_state();
 103                 }
 104         } while (curstate != STOPMACHINE_EXIT);
 105
 106         local_irq_enable();
 107 }
 108
 109 /* Callback for CPUs which aren't supposed to do anything. */
 110 static int chill(void *unused)
 111 {
 112         return 0;
 113 }
 114
 115 int stop_machine_create(void)
 116 {
 117         mutex_lock(&setup_lock);
 118         if (refcount)
 119                 goto done;
 120         stop_machine_wq = create_rt_workqueue("kstop");
 121         if (!stop_machine_wq)
 122                 goto err_out;
 123         stop_machine_work = alloc_percpu(struct work_struct);
 124         if (!stop_machine_work)
 125                 goto err_out;
 126 done:
 127         refcount++;
 128         mutex_unlock(&setup_lock);
 129         return 0;
 130
 131 err_out:
 132         if (stop_machine_wq)
 133                 destroy_workqueue(stop_machine_wq);
 134         mutex_unlock(&setup_lock);
 135         return -ENOMEM;
 136 }
 137 EXPORT_SYMBOL_GPL(stop_machine_create);
 138
 139 void stop_machine_destroy(void)
 140 {
 141         mutex_lock(&setup_lock);
 142         refcount--;
 143         if (refcount)
 144                 goto done;
 145         destroy_workqueue(stop_machine_wq);
 146         free_percpu(stop_machine_work);
 147 done:
 148         mutex_unlock(&setup_lock);
 149 }
 150 EXPORT_SYMBOL_GPL(stop_machine_destroy);
 151
 152 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
 153 {
 154         struct work_struct *sm_work;
 155         int i, ret;
 156
 157         /* Set up initial state. */
 158         mutex_lock(&lock);
 159         num_threads = num_online_cpus();
 160         active_cpus = cpus;
 161         active.fn = fn;
 162         active.data = data;
 163         active.fnret = 0;
 164         idle.fn = chill;
 165         idle.data = NULL;
 166
 167         set_state(STOPMACHINE_PREPARE);
 168
 169         /* Schedule the stop_cpu work on all cpus: hold this CPU so one
 170          * doesn't hit this CPU until we're ready. */
 171         get_cpu();
 172         for_each_online_cpu(i) {
 173                 sm_work = per_cpu_ptr(stop_machine_work, i);
 174                 INIT_WORK(sm_work, stop_cpu);
 175                 queue_work_on(i, stop_machine_wq, sm_work);
 176         }
 177         /* This will release the thread on our CPU. */
 178         put_cpu();
 179         flush_workqueue(stop_machine_wq);
 180         ret = active.fnret;
 181         mutex_unlock(&lock);
 182         return ret;
 183 }
 184
 185 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
 186 {
 187         int ret;
 188
 189         ret = stop_machine_create();
 190         if (ret)
 191                 return ret;
 192         /* No CPUs can come up or down during this. */
 193         get_online_cpus();
 194         ret = __stop_machine(fn, data, cpus);
 195         put_online_cpus();
 196         stop_machine_destroy();
 197         return ret;
 198 }
 199 EXPORT_SYMBOL_GPL(stop_machine);