drivers/cpuidle/cpuidle.c

   1 /*
   2  * cpuidle.c - core cpuidle infrastructure
   3  *
   4  * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
   5  *               Shaohua Li <shaohua.li@intel.com>
   6  *               Adam Belay <abelay@novell.com>
   7  *
   8  * This code is licenced under the GPL.
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/mutex.h>
  13 #include <linux/sched.h>
  14 #include <linux/notifier.h>
  15 #include <linux/pm_qos_params.h>
  16 #include <linux/cpu.h>
  17 #include <linux/cpuidle.h>
  18 #include <linux/ktime.h>
  19
  20 #include "cpuidle.h"
  21
  22 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
  23
  24 DEFINE_MUTEX(cpuidle_lock);
  25 LIST_HEAD(cpuidle_detected_devices);
  26 static void (*pm_idle_old)(void);
  27
  28 static int enabled_devices;
  29
  30 #if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT)
  31 static void cpuidle_kick_cpus(void)
  32 {
  33         cpu_idle_wait();
  34 }
  35 #elif defined(CONFIG_SMP)
  36 # error "Arch needs cpu_idle_wait() equivalent here"
  37 #else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */
  38 static void cpuidle_kick_cpus(void) {}
  39 #endif
  40
  41 static int __cpuidle_register_device(struct cpuidle_device *dev);
  42
  43 /**
  44  * cpuidle_idle_call - the main idle loop
  45  *
  46  * NOTE: no locks or semaphores should be used here
  47  */
  48 static void cpuidle_idle_call(void)
  49 {
  50         struct cpuidle_device *dev = __get_cpu_var(cpuidle_devices);
  51         struct cpuidle_state *target_state;
  52         int next_state;
  53
  54         /* check if the device is ready */
  55         if (!dev || !dev->enabled) {
  56                 if (pm_idle_old)
  57                         pm_idle_old();
  58                 else
  59                         local_irq_enable();
  60                 return;
  61         }
  62
  63         /* ask the governor for the next state */
  64         next_state = cpuidle_curr_governor->select(dev);
  65         if (need_resched())
  66                 return;
  67         target_state = &dev->states[next_state];
  68
  69         /* enter the state and update stats */
  70         dev->last_state = target_state;
  71         dev->last_residency = target_state->enter(dev, target_state);
  72         if (dev->last_state)
  73                 target_state = dev->last_state;
  74
  75         target_state->time += (unsigned long long)dev->last_residency;
  76         target_state->usage++;
  77
  78         /* give the governor an opportunity to reflect on the outcome */
  79         if (cpuidle_curr_governor->reflect)
  80                 cpuidle_curr_governor->reflect(dev);
  81 }
  82
  83 /**
  84  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
  85  */
  86 void cpuidle_install_idle_handler(void)
  87 {
  88         if (enabled_devices && (pm_idle != cpuidle_idle_call)) {
  89                 /* Make sure all changes finished before we switch to new idle */
  90                 smp_wmb();
  91                 pm_idle = cpuidle_idle_call;
  92         }
  93 }
  94
  95 /**
  96  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
  97  */
  98 void cpuidle_uninstall_idle_handler(void)
  99 {
 100         if (enabled_devices && pm_idle_old && (pm_idle != pm_idle_old)) {
 101                 pm_idle = pm_idle_old;
 102                 cpuidle_kick_cpus();
 103         }
 104 }
 105
 106 /**
 107  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
 108  */
 109 void cpuidle_pause_and_lock(void)
 110 {
 111         mutex_lock(&cpuidle_lock);
 112         cpuidle_uninstall_idle_handler();
 113 }
 114
 115 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
 116
 117 /**
 118  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
 119  */
 120 void cpuidle_resume_and_unlock(void)
 121 {
 122         cpuidle_install_idle_handler();
 123         mutex_unlock(&cpuidle_lock);
 124 }
 125
 126 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
 127
 128 /**
 129  * cpuidle_enable_device - enables idle PM for a CPU
 130  * @dev: the CPU
 131  *
 132  * This function must be called between cpuidle_pause_and_lock and
 133  * cpuidle_resume_and_unlock when used externally.
 134  */
 135 int cpuidle_enable_device(struct cpuidle_device *dev)
 136 {
 137         int ret, i;
 138
 139         if (dev->enabled)
 140                 return 0;
 141         if (!cpuidle_curr_driver || !cpuidle_curr_governor)
 142                 return -EIO;
 143         if (!dev->state_count)
 144                 return -EINVAL;
 145
 146         if (dev->registered == 0) {
 147                 ret = __cpuidle_register_device(dev);
 148                 if (ret)
 149                         return ret;
 150         }
 151
 152         if ((ret = cpuidle_add_state_sysfs(dev)))
 153                 return ret;
 154
 155         if (cpuidle_curr_governor->enable &&
 156             (ret = cpuidle_curr_governor->enable(dev)))
 157                 goto fail_sysfs;
 158
 159         for (i = 0; i < dev->state_count; i++) {
 160                 dev->states[i].usage = 0;
 161                 dev->states[i].time = 0;
 162         }
 163         dev->last_residency = 0;
 164         dev->last_state = NULL;
 165
 166         smp_wmb();
 167
 168         dev->enabled = 1;
 169
 170         enabled_devices++;
 171         return 0;
 172
 173 fail_sysfs:
 174         cpuidle_remove_state_sysfs(dev);
 175
 176         return ret;
 177 }
 178
 179 EXPORT_SYMBOL_GPL(cpuidle_enable_device);
 180
 181 /**
 182  * cpuidle_disable_device - disables idle PM for a CPU
 183  * @dev: the CPU
 184  *
 185  * This function must be called between cpuidle_pause_and_lock and
 186  * cpuidle_resume_and_unlock when used externally.
 187  */
 188 void cpuidle_disable_device(struct cpuidle_device *dev)
 189 {
 190         if (!dev->enabled)
 191                 return;
 192         if (!cpuidle_curr_driver || !cpuidle_curr_governor)
 193                 return;
 194
 195         dev->enabled = 0;
 196
 197         if (cpuidle_curr_governor->disable)
 198                 cpuidle_curr_governor->disable(dev);
 199
 200         cpuidle_remove_state_sysfs(dev);
 201         enabled_devices--;
 202 }
 203
 204 EXPORT_SYMBOL_GPL(cpuidle_disable_device);
 205
 206 #ifdef CONFIG_ARCH_HAS_CPU_RELAX
 207 static int poll_idle(struct cpuidle_device *dev, struct cpuidle_state *st)
 208 {
 209         ktime_t t1, t2;
 210         s64 diff;
 211         int ret;
 212
 213         t1 = ktime_get();
 214         local_irq_enable();
 215         while (!need_resched())
 216                 cpu_relax();
 217
 218         t2 = ktime_get();
 219         diff = ktime_to_us(ktime_sub(t2, t1));
 220         if (diff > INT_MAX)
 221                 diff = INT_MAX;
 222
 223         ret = (int) diff;
 224         return ret;
 225 }
 226
 227 static void poll_idle_init(struct cpuidle_device *dev)
 228 {
 229         struct cpuidle_state *state = &dev->states[0];
 230
 231         cpuidle_set_statedata(state, NULL);
 232
 233         snprintf(state->name, CPUIDLE_NAME_LEN, "C0");
 234         snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
 235         state->exit_latency = 0;
 236         state->target_residency = 0;
 237         state->power_usage = -1;
 238         state->flags = CPUIDLE_FLAG_POLL;
 239         state->enter = poll_idle;
 240 }
 241 #else
 242 static void poll_idle_init(struct cpuidle_device *dev) {}
 243 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */
 244
 245 /**
 246  * __cpuidle_register_device - internal register function called before register
 247  * and enable routines
 248  * @dev: the cpu
 249  *
 250  * cpuidle_lock mutex must be held before this is called
 251  */
 252 static int __cpuidle_register_device(struct cpuidle_device *dev)
 253 {
 254         int ret;
 255         struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
 256
 257         if (!sys_dev)
 258                 return -EINVAL;
 259         if (!try_module_get(cpuidle_curr_driver->owner))
 260                 return -EINVAL;
 261
 262         init_completion(&dev->kobj_unregister);
 263
 264         poll_idle_init(dev);
 265
 266         per_cpu(cpuidle_devices, dev->cpu) = dev;
 267         list_add(&dev->device_list, &cpuidle_detected_devices);
 268         if ((ret = cpuidle_add_sysfs(sys_dev))) {
 269                 module_put(cpuidle_curr_driver->owner);
 270                 return ret;
 271         }
 272
 273         dev->registered = 1;
 274         return 0;
 275 }
 276
 277 /**
 278  * cpuidle_register_device - registers a CPU's idle PM feature
 279  * @dev: the cpu
 280  */
 281 int cpuidle_register_device(struct cpuidle_device *dev)
 282 {
 283         int ret;
 284
 285         mutex_lock(&cpuidle_lock);
 286
 287         if ((ret = __cpuidle_register_device(dev))) {
 288                 mutex_unlock(&cpuidle_lock);
 289                 return ret;
 290         }
 291
 292         cpuidle_enable_device(dev);
 293         cpuidle_install_idle_handler();
 294
 295         mutex_unlock(&cpuidle_lock);
 296
 297         return 0;
 298
 299 }
 300
 301 EXPORT_SYMBOL_GPL(cpuidle_register_device);
 302
 303 /**
 304  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
 305  * @dev: the cpu
 306  */
 307 void cpuidle_unregister_device(struct cpuidle_device *dev)
 308 {
 309         struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
 310
 311         if (dev->registered == 0)
 312                 return;
 313
 314         cpuidle_pause_and_lock();
 315
 316         cpuidle_disable_device(dev);
 317
 318         cpuidle_remove_sysfs(sys_dev);
 319         list_del(&dev->device_list);
 320         wait_for_completion(&dev->kobj_unregister);
 321         per_cpu(cpuidle_devices, dev->cpu) = NULL;
 322
 323         cpuidle_resume_and_unlock();
 324
 325         module_put(cpuidle_curr_driver->owner);
 326 }
 327
 328 EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
 329
 330 #ifdef CONFIG_SMP
 331
 332 static void smp_callback(void *v)
 333 {
 334         /* we already woke the CPU up, nothing more to do */
 335 }
 336
 337 /*
 338  * This function gets called when a part of the kernel has a new latency
 339  * requirement.  This means we need to get all processors out of their C-state,
 340  * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
 341  * wakes them all right up.
 342  */
 343 static int cpuidle_latency_notify(struct notifier_block *b,
 344                 unsigned long l, void *v)
 345 {
 346         smp_call_function(smp_callback, NULL, 1);
 347         return NOTIFY_OK;
 348 }
 349
 350 static struct notifier_block cpuidle_latency_notifier = {
 351         .notifier_call = cpuidle_latency_notify,
 352 };
 353
 354 static inline void latency_notifier_init(struct notifier_block *n)
 355 {
 356         pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
 357 }
 358
 359 #else /* CONFIG_SMP */
 360
 361 #define latency_notifier_init(x) do { } while (0)
 362
 363 #endif /* CONFIG_SMP */
 364
 365 /**
 366  * cpuidle_init - core initializer
 367  */
 368 static int __init cpuidle_init(void)
 369 {
 370         int ret;
 371
 372         pm_idle_old = pm_idle;
 373
 374         ret = cpuidle_add_class_sysfs(&cpu_sysdev_class);
 375         if (ret)
 376                 return ret;
 377
 378         latency_notifier_init(&cpuidle_latency_notifier);
 379
 380         return 0;
 381 }
 382
 383 core_initcall(cpuidle_init);