2 * sched_clock for unstable cpu clocks
4 * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
6 * Updates and enhancements:
7 * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com>
10 * Ingo Molnar <mingo@redhat.com>
11 * Guillaume Chazarain <guichaz@gmail.com>
13 * Create a semi stable clock from a mixture of other events, including:
16 * - explicit idle events
18 * We use gtod as base and the unstable clock deltas. The deltas are filtered,
19 * making it monotonic and keeping it within an expected window.
21 * Furthermore, explicit sleep and wakeup hooks allow us to account for time
22 * that is otherwise invisible (TSC gets stopped).
24 * The clock: sched_clock_cpu() is monotonic per cpu, and should be somewhat
25 * consistent between cpus (never more than 2 jiffies difference).
27 #include <linux/spinlock.h>
28 #include <linux/hardirq.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/ktime.h>
32 #include <linux/sched.h>
35 * Scheduler clock - returns current time in nanosec units.
36 * This is default implementation.
37 * Architectures and sub-architectures can override this.
39 unsigned long long __attribute__((weak
)) sched_clock(void)
41 return (unsigned long long)jiffies
* (NSEC_PER_SEC
/ HZ
);
44 static __read_mostly
int sched_clock_running
;
46 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
47 __read_mostly
int sched_clock_stable
;
49 static const int sched_clock_stable
= 1;
52 struct sched_clock_data
{
54 * Raw spinlock - this is a special case: this might be called
55 * from within instrumentation code so we dont want to do any
56 * instrumentation ourselves.
65 static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data
, sched_clock_data
);
67 static inline struct sched_clock_data
*this_scd(void)
69 return &__get_cpu_var(sched_clock_data
);
72 static inline struct sched_clock_data
*cpu_sdc(int cpu
)
74 return &per_cpu(sched_clock_data
, cpu
);
77 void sched_clock_init(void)
79 u64 ktime_now
= ktime_to_ns(ktime_get());
82 for_each_possible_cpu(cpu
) {
83 struct sched_clock_data
*scd
= cpu_sdc(cpu
);
85 scd
->lock
= (raw_spinlock_t
)__RAW_SPIN_LOCK_UNLOCKED
;
87 scd
->tick_gtod
= ktime_now
;
88 scd
->clock
= ktime_now
;
91 sched_clock_running
= 1;
95 * min, max except they take wrapping into account
98 static inline u64
wrap_min(u64 x
, u64 y
)
100 return (s64
)(x
- y
) < 0 ? x
: y
;
103 static inline u64
wrap_max(u64 x
, u64 y
)
105 return (s64
)(x
- y
) > 0 ? x
: y
;
109 * update the percpu scd from the raw @now value
111 * - filter out backward motion
112 * - use the GTOD tick value to create a window to filter crazy TSC values
114 static u64
__update_sched_clock(struct sched_clock_data
*scd
, u64 now
)
116 s64 delta
= now
- scd
->tick_raw
;
117 u64 clock
, min_clock
, max_clock
;
119 WARN_ON_ONCE(!irqs_disabled());
121 if (unlikely(delta
< 0))
124 if (unlikely(!sched_clock_running
))
128 * scd->clock = clamp(scd->tick_gtod + delta,
129 * max(scd->tick_gtod, scd->clock),
130 * scd->tick_gtod + TICK_NSEC);
133 clock
= scd
->tick_gtod
+ delta
;
134 min_clock
= wrap_max(scd
->tick_gtod
, scd
->clock
);
135 max_clock
= wrap_max(scd
->clock
, scd
->tick_gtod
+ TICK_NSEC
);
137 clock
= wrap_max(clock
, min_clock
);
138 clock
= wrap_min(clock
, max_clock
);
145 static void lock_double_clock(struct sched_clock_data
*data1
,
146 struct sched_clock_data
*data2
)
149 __raw_spin_lock(&data1
->lock
);
150 __raw_spin_lock(&data2
->lock
);
152 __raw_spin_lock(&data2
->lock
);
153 __raw_spin_lock(&data1
->lock
);
157 u64
sched_clock_cpu(int cpu
)
159 u64 now
, clock
, this_clock
, remote_clock
;
160 struct sched_clock_data
*scd
;
162 if (sched_clock_stable
)
163 return sched_clock();
168 * Normally this is not called in NMI context - but if it is,
169 * trying to do any locking here is totally lethal.
171 if (unlikely(in_nmi()))
174 if (unlikely(!sched_clock_running
))
177 WARN_ON_ONCE(!irqs_disabled());
180 if (cpu
!= raw_smp_processor_id()) {
181 struct sched_clock_data
*my_scd
= this_scd();
183 lock_double_clock(scd
, my_scd
);
185 this_clock
= __update_sched_clock(my_scd
, now
);
186 remote_clock
= scd
->clock
;
189 * Use the opportunity that we have both locks
190 * taken to couple the two clocks: we take the
191 * larger time as the latest time for both
192 * runqueues. (this creates monotonic movement)
194 if (likely((s64
)(remote_clock
- this_clock
) < 0)) {
199 * Should be rare, but possible:
201 clock
= remote_clock
;
202 my_scd
->clock
= remote_clock
;
205 __raw_spin_unlock(&my_scd
->lock
);
207 __raw_spin_lock(&scd
->lock
);
208 clock
= __update_sched_clock(scd
, now
);
211 __raw_spin_unlock(&scd
->lock
);
216 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
218 void sched_clock_tick(void)
220 struct sched_clock_data
*scd
= this_scd();
223 if (unlikely(!sched_clock_running
))
226 WARN_ON_ONCE(!irqs_disabled());
228 now_gtod
= ktime_to_ns(ktime_get());
231 __raw_spin_lock(&scd
->lock
);
233 scd
->tick_gtod
= now_gtod
;
234 __update_sched_clock(scd
, now
);
235 __raw_spin_unlock(&scd
->lock
);
239 * We are going deep-idle (irqs are disabled):
241 void sched_clock_idle_sleep_event(void)
243 sched_clock_cpu(smp_processor_id());
245 EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event
);
248 * We just idled delta nanoseconds (called with irqs disabled):
250 void sched_clock_idle_wakeup_event(u64 delta_ns
)
252 if (timekeeping_suspended
)
256 touch_softlockup_watchdog();
258 EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event
);
260 #endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
262 unsigned long long cpu_clock(int cpu
)
264 unsigned long long clock
;
267 local_irq_save(flags
);
268 clock
= sched_clock_cpu(cpu
);
269 local_irq_restore(flags
);
273 EXPORT_SYMBOL_GPL(cpu_clock
);