2 * drivers/cpufreq/cpufreq_ondemand.c
4 * Copyright (C) 2001 Russell King
5 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
6 * Jun Nakajima <jun.nakajima@intel.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/cpufreq.h>
17 #include <linux/cpu.h>
18 #include <linux/jiffies.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/mutex.h>
23 * dbs is used in this file as a shortform for demandbased switching
24 * It helps to keep variable names smaller, simpler
27 #define DEF_FREQUENCY_UP_THRESHOLD (80)
28 #define MIN_FREQUENCY_UP_THRESHOLD (11)
29 #define MAX_FREQUENCY_UP_THRESHOLD (100)
32 * The polling frequency of this governor depends on the capability of
33 * the processor. Default polling frequency is 1000 times the transition
34 * latency of the processor. The governor will work on any processor with
35 * transition latency <= 10mS, using appropriate sampling
37 * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL)
38 * this governor will not work.
39 * All times here are in uS.
41 static unsigned int def_sampling_rate
;
42 #define MIN_SAMPLING_RATE_RATIO (2)
43 /* for correct statistics, we need at least 10 ticks between each measure */
44 #define MIN_STAT_SAMPLING_RATE (MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10))
45 #define MIN_SAMPLING_RATE (def_sampling_rate / MIN_SAMPLING_RATE_RATIO)
46 #define MAX_SAMPLING_RATE (500 * def_sampling_rate)
47 #define DEF_SAMPLING_RATE_LATENCY_MULTIPLIER (1000)
48 #define TRANSITION_LATENCY_LIMIT (10 * 1000)
50 static void do_dbs_timer(void *data
);
52 struct cpu_dbs_info_s
{
53 cputime64_t prev_cpu_idle
;
54 cputime64_t prev_cpu_wall
;
55 struct cpufreq_policy
*cur_policy
;
56 struct work_struct work
;
59 static DEFINE_PER_CPU(struct cpu_dbs_info_s
, cpu_dbs_info
);
61 static unsigned int dbs_enable
; /* number of CPUs using this policy */
64 * DEADLOCK ALERT! There is a ordering requirement between cpu_hotplug
65 * lock and dbs_mutex. cpu_hotplug lock should always be held before
66 * dbs_mutex. If any function that can potentially take cpu_hotplug lock
67 * (like __cpufreq_driver_target()) is being called with dbs_mutex taken, then
68 * cpu_hotplug lock should be taken before that. Note that cpu_hotplug lock
69 * is recursive for the same process. -Venki
71 static DEFINE_MUTEX(dbs_mutex
);
73 static struct workqueue_struct
*kondemand_wq
;
76 unsigned int sampling_rate
;
77 unsigned int up_threshold
;
78 unsigned int ignore_nice
;
81 static struct dbs_tuners dbs_tuners_ins
= {
82 .up_threshold
= DEF_FREQUENCY_UP_THRESHOLD
,
86 static inline cputime64_t
get_cpu_idle_time(unsigned int cpu
)
90 retval
= cputime64_add(kstat_cpu(cpu
).cpustat
.idle
,
91 kstat_cpu(cpu
).cpustat
.iowait
);
93 if (dbs_tuners_ins
.ignore_nice
)
94 retval
= cputime64_add(retval
, kstat_cpu(cpu
).cpustat
.nice
);
99 /************************** sysfs interface ************************/
100 static ssize_t
show_sampling_rate_max(struct cpufreq_policy
*policy
, char *buf
)
102 return sprintf (buf
, "%u\n", MAX_SAMPLING_RATE
);
105 static ssize_t
show_sampling_rate_min(struct cpufreq_policy
*policy
, char *buf
)
107 return sprintf (buf
, "%u\n", MIN_SAMPLING_RATE
);
110 #define define_one_ro(_name) \
111 static struct freq_attr _name = \
112 __ATTR(_name, 0444, show_##_name, NULL)
114 define_one_ro(sampling_rate_max
);
115 define_one_ro(sampling_rate_min
);
117 /* cpufreq_ondemand Governor Tunables */
118 #define show_one(file_name, object) \
119 static ssize_t show_##file_name \
120 (struct cpufreq_policy *unused, char *buf) \
122 return sprintf(buf, "%u\n", dbs_tuners_ins.object); \
124 show_one(sampling_rate
, sampling_rate
);
125 show_one(up_threshold
, up_threshold
);
126 show_one(ignore_nice_load
, ignore_nice
);
128 static ssize_t
store_sampling_rate(struct cpufreq_policy
*unused
,
129 const char *buf
, size_t count
)
133 ret
= sscanf(buf
, "%u", &input
);
135 mutex_lock(&dbs_mutex
);
136 if (ret
!= 1 || input
> MAX_SAMPLING_RATE
|| input
< MIN_SAMPLING_RATE
) {
137 mutex_unlock(&dbs_mutex
);
141 dbs_tuners_ins
.sampling_rate
= input
;
142 mutex_unlock(&dbs_mutex
);
147 static ssize_t
store_up_threshold(struct cpufreq_policy
*unused
,
148 const char *buf
, size_t count
)
152 ret
= sscanf(buf
, "%u", &input
);
154 mutex_lock(&dbs_mutex
);
155 if (ret
!= 1 || input
> MAX_FREQUENCY_UP_THRESHOLD
||
156 input
< MIN_FREQUENCY_UP_THRESHOLD
) {
157 mutex_unlock(&dbs_mutex
);
161 dbs_tuners_ins
.up_threshold
= input
;
162 mutex_unlock(&dbs_mutex
);
167 static ssize_t
store_ignore_nice_load(struct cpufreq_policy
*policy
,
168 const char *buf
, size_t count
)
175 ret
= sscanf(buf
, "%u", &input
);
182 mutex_lock(&dbs_mutex
);
183 if ( input
== dbs_tuners_ins
.ignore_nice
) { /* nothing to do */
184 mutex_unlock(&dbs_mutex
);
187 dbs_tuners_ins
.ignore_nice
= input
;
189 /* we need to re-evaluate prev_cpu_idle */
190 for_each_online_cpu(j
) {
191 struct cpu_dbs_info_s
*dbs_info
;
192 dbs_info
= &per_cpu(cpu_dbs_info
, j
);
193 dbs_info
->prev_cpu_idle
= get_cpu_idle_time(j
);
194 dbs_info
->prev_cpu_wall
= get_jiffies_64();
196 mutex_unlock(&dbs_mutex
);
201 #define define_one_rw(_name) \
202 static struct freq_attr _name = \
203 __ATTR(_name, 0644, show_##_name, store_##_name)
205 define_one_rw(sampling_rate
);
206 define_one_rw(up_threshold
);
207 define_one_rw(ignore_nice_load
);
209 static struct attribute
* dbs_attributes
[] = {
210 &sampling_rate_max
.attr
,
211 &sampling_rate_min
.attr
,
214 &ignore_nice_load
.attr
,
218 static struct attribute_group dbs_attr_group
= {
219 .attrs
= dbs_attributes
,
223 /************************** sysfs end ************************/
225 static void dbs_check_cpu(struct cpu_dbs_info_s
*this_dbs_info
)
227 unsigned int idle_ticks
, total_ticks
;
229 cputime64_t cur_jiffies
;
231 struct cpufreq_policy
*policy
;
234 if (!this_dbs_info
->enable
)
237 policy
= this_dbs_info
->cur_policy
;
238 cur_jiffies
= jiffies64_to_cputime64(get_jiffies_64());
239 total_ticks
= (unsigned int) cputime64_sub(cur_jiffies
,
240 this_dbs_info
->prev_cpu_wall
);
241 this_dbs_info
->prev_cpu_wall
= cur_jiffies
;
243 * Every sampling_rate, we check, if current idle time is less
244 * than 20% (default), then we try to increase frequency
245 * Every sampling_rate, we look for a the lowest
246 * frequency which can sustain the load while keeping idle time over
247 * 30%. If such a frequency exist, we try to decrease to this frequency.
249 * Any frequency increase takes it to the maximum frequency.
250 * Frequency reduction happens at minimum steps of
251 * 5% (default) of current frequency
255 idle_ticks
= UINT_MAX
;
256 for_each_cpu_mask(j
, policy
->cpus
) {
257 cputime64_t total_idle_ticks
;
258 unsigned int tmp_idle_ticks
;
259 struct cpu_dbs_info_s
*j_dbs_info
;
261 j_dbs_info
= &per_cpu(cpu_dbs_info
, j
);
262 total_idle_ticks
= get_cpu_idle_time(j
);
263 tmp_idle_ticks
= (unsigned int) cputime64_sub(total_idle_ticks
,
264 j_dbs_info
->prev_cpu_idle
);
265 j_dbs_info
->prev_cpu_idle
= total_idle_ticks
;
267 if (tmp_idle_ticks
< idle_ticks
)
268 idle_ticks
= tmp_idle_ticks
;
270 load
= (100 * (total_ticks
- idle_ticks
)) / total_ticks
;
272 /* Check for frequency increase */
273 if (load
> dbs_tuners_ins
.up_threshold
) {
274 /* if we are already at full speed then break out early */
275 if (policy
->cur
== policy
->max
)
278 __cpufreq_driver_target(policy
, policy
->max
,
283 /* Check for frequency decrease */
284 /* if we cannot reduce the frequency anymore, break out early */
285 if (policy
->cur
== policy
->min
)
289 * The optimal frequency is the frequency that is the lowest that
290 * can support the current CPU usage without triggering the up
291 * policy. To be safe, we focus 10 points under the threshold.
293 if (load
< (dbs_tuners_ins
.up_threshold
- 10)) {
294 unsigned int freq_next
;
295 freq_next
= (policy
->cur
* load
) /
296 (dbs_tuners_ins
.up_threshold
- 10);
298 __cpufreq_driver_target(policy
, freq_next
, CPUFREQ_RELATION_L
);
302 static void do_dbs_timer(void *data
)
304 unsigned int cpu
= smp_processor_id();
305 struct cpu_dbs_info_s
*dbs_info
= &per_cpu(cpu_dbs_info
, cpu
);
307 dbs_check_cpu(dbs_info
);
308 queue_delayed_work_on(cpu
, kondemand_wq
, &dbs_info
->work
,
309 usecs_to_jiffies(dbs_tuners_ins
.sampling_rate
));
312 static inline void dbs_timer_init(unsigned int cpu
)
314 struct cpu_dbs_info_s
*dbs_info
= &per_cpu(cpu_dbs_info
, cpu
);
316 INIT_WORK(&dbs_info
->work
, do_dbs_timer
, 0);
317 queue_delayed_work_on(cpu
, kondemand_wq
, &dbs_info
->work
,
318 usecs_to_jiffies(dbs_tuners_ins
.sampling_rate
));
322 static inline void dbs_timer_exit(unsigned int cpu
)
324 struct cpu_dbs_info_s
*dbs_info
= &per_cpu(cpu_dbs_info
, cpu
);
326 cancel_rearming_delayed_workqueue(kondemand_wq
, &dbs_info
->work
);
329 static int cpufreq_governor_dbs(struct cpufreq_policy
*policy
,
332 unsigned int cpu
= policy
->cpu
;
333 struct cpu_dbs_info_s
*this_dbs_info
;
336 this_dbs_info
= &per_cpu(cpu_dbs_info
, cpu
);
339 case CPUFREQ_GOV_START
:
340 if ((!cpu_online(cpu
)) || (!policy
->cur
))
343 if (policy
->cpuinfo
.transition_latency
>
344 (TRANSITION_LATENCY_LIMIT
* 1000)) {
345 printk(KERN_WARNING
"ondemand governor failed to load "
346 "due to too long transition latency\n");
349 if (this_dbs_info
->enable
) /* Already enabled */
352 mutex_lock(&dbs_mutex
);
354 if (dbs_enable
== 1) {
355 kondemand_wq
= create_workqueue("kondemand");
357 printk(KERN_ERR
"Creation of kondemand failed\n");
359 mutex_unlock(&dbs_mutex
);
363 for_each_cpu_mask(j
, policy
->cpus
) {
364 struct cpu_dbs_info_s
*j_dbs_info
;
365 j_dbs_info
= &per_cpu(cpu_dbs_info
, j
);
366 j_dbs_info
->cur_policy
= policy
;
368 j_dbs_info
->prev_cpu_idle
= get_cpu_idle_time(j
);
369 j_dbs_info
->prev_cpu_wall
= get_jiffies_64();
371 this_dbs_info
->enable
= 1;
372 sysfs_create_group(&policy
->kobj
, &dbs_attr_group
);
374 * Start the timerschedule work, when this governor
375 * is used for first time
377 if (dbs_enable
== 1) {
378 unsigned int latency
;
379 /* policy latency is in nS. Convert it to uS first */
380 latency
= policy
->cpuinfo
.transition_latency
/ 1000;
384 def_sampling_rate
= latency
*
385 DEF_SAMPLING_RATE_LATENCY_MULTIPLIER
;
387 if (def_sampling_rate
< MIN_STAT_SAMPLING_RATE
)
388 def_sampling_rate
= MIN_STAT_SAMPLING_RATE
;
390 dbs_tuners_ins
.sampling_rate
= def_sampling_rate
;
392 dbs_timer_init(policy
->cpu
);
394 mutex_unlock(&dbs_mutex
);
397 case CPUFREQ_GOV_STOP
:
398 mutex_lock(&dbs_mutex
);
399 dbs_timer_exit(policy
->cpu
);
400 this_dbs_info
->enable
= 0;
401 sysfs_remove_group(&policy
->kobj
, &dbs_attr_group
);
404 destroy_workqueue(kondemand_wq
);
406 mutex_unlock(&dbs_mutex
);
410 case CPUFREQ_GOV_LIMITS
:
412 mutex_lock(&dbs_mutex
);
413 if (policy
->max
< this_dbs_info
->cur_policy
->cur
)
414 __cpufreq_driver_target(this_dbs_info
->cur_policy
,
417 else if (policy
->min
> this_dbs_info
->cur_policy
->cur
)
418 __cpufreq_driver_target(this_dbs_info
->cur_policy
,
421 mutex_unlock(&dbs_mutex
);
422 unlock_cpu_hotplug();
428 static struct cpufreq_governor cpufreq_gov_dbs
= {
430 .governor
= cpufreq_governor_dbs
,
431 .owner
= THIS_MODULE
,
434 static int __init
cpufreq_gov_dbs_init(void)
436 return cpufreq_register_governor(&cpufreq_gov_dbs
);
439 static void __exit
cpufreq_gov_dbs_exit(void)
441 cpufreq_unregister_governor(&cpufreq_gov_dbs
);
445 MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
446 MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
447 MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
448 "Low Latency Frequency Transition capable processors");
449 MODULE_LICENSE("GPL");
451 module_init(cpufreq_gov_dbs_init
);
452 module_exit(cpufreq_gov_dbs_exit
);