2 * Xen time implementation.
4 * This is implemented in terms of a clocksource driver which uses
5 * the hypervisor clock as a nanosecond timebase, and a clockevent
6 * driver which uses the hypervisor's timer mechanism.
8 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
10 #include <linux/kernel.h>
11 #include <linux/interrupt.h>
12 #include <linux/clocksource.h>
13 #include <linux/clockchips.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/math64.h>
16 #include <linux/gfp.h>
18 #include <asm/pvclock.h>
19 #include <asm/xen/hypervisor.h>
20 #include <asm/xen/hypercall.h>
22 #include <xen/events.h>
23 #include <xen/features.h>
24 #include <xen/interface/xen.h>
25 #include <xen/interface/vcpu.h>
31 /* Xen may fire a timer up to this many ns early */
32 #define TIMER_SLOP 100000
33 #define NS_PER_TICK (1000000000LL / HZ)
35 /* runstate info updated by Xen */
36 static DEFINE_PER_CPU(struct vcpu_runstate_info
, xen_runstate
);
38 /* snapshots of runstate info */
39 static DEFINE_PER_CPU(struct vcpu_runstate_info
, xen_runstate_snapshot
);
41 /* unused ns of stolen and blocked time */
42 static DEFINE_PER_CPU(u64
, xen_residual_stolen
);
43 static DEFINE_PER_CPU(u64
, xen_residual_blocked
);
45 /* return an consistent snapshot of 64-bit time/counter value */
46 static u64
get64(const u64
*p
)
50 if (BITS_PER_LONG
< 64) {
55 * Read high then low, and then make sure high is
56 * still the same; this will only loop if low wraps
57 * and carries into high.
58 * XXX some clean way to make this endian-proof?
65 } while (p32
[1] != h
);
67 ret
= (((u64
)h
) << 32) | l
;
77 static void get_runstate_snapshot(struct vcpu_runstate_info
*res
)
80 struct vcpu_runstate_info
*state
;
82 BUG_ON(preemptible());
84 state
= &__get_cpu_var(xen_runstate
);
87 * The runstate info is always updated by the hypervisor on
88 * the current CPU, so there's no need to use anything
89 * stronger than a compiler barrier when fetching it.
92 state_time
= get64(&state
->state_entry_time
);
96 } while (get64(&state
->state_entry_time
) != state_time
);
99 /* return true when a vcpu could run but has no real cpu to run on */
100 bool xen_vcpu_stolen(int vcpu
)
102 return per_cpu(xen_runstate
, vcpu
).state
== RUNSTATE_runnable
;
105 void xen_setup_runstate_info(int cpu
)
107 struct vcpu_register_runstate_memory_area area
;
109 area
.addr
.v
= &per_cpu(xen_runstate
, cpu
);
111 if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area
,
116 static void do_stolen_accounting(void)
118 struct vcpu_runstate_info state
;
119 struct vcpu_runstate_info
*snap
;
120 s64 blocked
, runnable
, offline
, stolen
;
123 get_runstate_snapshot(&state
);
125 WARN_ON(state
.state
!= RUNSTATE_running
);
127 snap
= &__get_cpu_var(xen_runstate_snapshot
);
129 /* work out how much time the VCPU has not been runn*ing* */
130 blocked
= state
.time
[RUNSTATE_blocked
] - snap
->time
[RUNSTATE_blocked
];
131 runnable
= state
.time
[RUNSTATE_runnable
] - snap
->time
[RUNSTATE_runnable
];
132 offline
= state
.time
[RUNSTATE_offline
] - snap
->time
[RUNSTATE_offline
];
136 /* Add the appropriate number of ticks of stolen time,
137 including any left-overs from last time. */
138 stolen
= runnable
+ offline
+ __this_cpu_read(xen_residual_stolen
);
143 ticks
= iter_div_u64_rem(stolen
, NS_PER_TICK
, &stolen
);
144 __this_cpu_write(xen_residual_stolen
, stolen
);
145 account_steal_ticks(ticks
);
147 /* Add the appropriate number of ticks of blocked time,
148 including any left-overs from last time. */
149 blocked
+= __this_cpu_read(xen_residual_blocked
);
154 ticks
= iter_div_u64_rem(blocked
, NS_PER_TICK
, &blocked
);
155 __this_cpu_write(xen_residual_blocked
, blocked
);
156 account_idle_ticks(ticks
);
159 /* Get the TSC speed from Xen */
160 static unsigned long xen_tsc_khz(void)
162 struct pvclock_vcpu_time_info
*info
=
163 &HYPERVISOR_shared_info
->vcpu_info
[0].time
;
165 return pvclock_tsc_khz(info
);
168 cycle_t
xen_clocksource_read(void)
170 struct pvclock_vcpu_time_info
*src
;
173 src
= &get_cpu_var(xen_vcpu
)->time
;
174 ret
= pvclock_clocksource_read(src
);
175 put_cpu_var(xen_vcpu
);
179 static cycle_t
xen_clocksource_get_cycles(struct clocksource
*cs
)
181 return xen_clocksource_read();
184 static void xen_read_wallclock(struct timespec
*ts
)
186 struct shared_info
*s
= HYPERVISOR_shared_info
;
187 struct pvclock_wall_clock
*wall_clock
= &(s
->wc
);
188 struct pvclock_vcpu_time_info
*vcpu_time
;
190 vcpu_time
= &get_cpu_var(xen_vcpu
)->time
;
191 pvclock_read_wallclock(wall_clock
, vcpu_time
, ts
);
192 put_cpu_var(xen_vcpu
);
195 static unsigned long xen_get_wallclock(void)
199 xen_read_wallclock(&ts
);
203 static int xen_set_wallclock(unsigned long now
)
205 /* do nothing for domU */
209 static struct clocksource xen_clocksource __read_mostly
= {
212 .read
= xen_clocksource_get_cycles
,
214 .mult
= 1<<XEN_SHIFT
, /* time directly in nanoseconds */
216 .flags
= CLOCK_SOURCE_IS_CONTINUOUS
,
220 Xen clockevent implementation
222 Xen has two clockevent implementations:
224 The old timer_op one works with all released versions of Xen prior
225 to version 3.0.4. This version of the hypervisor provides a
226 single-shot timer with nanosecond resolution. However, sharing the
227 same event channel is a 100Hz tick which is delivered while the
228 vcpu is running. We don't care about or use this tick, but it will
229 cause the core time code to think the timer fired too soon, and
230 will end up resetting it each time. It could be filtered, but
231 doing so has complications when the ktime clocksource is not yet
232 the xen clocksource (ie, at boot time).
234 The new vcpu_op-based timer interface allows the tick timer period
235 to be changed or turned off. The tick timer is not useful as a
236 periodic timer because events are only delivered to running vcpus.
237 The one-shot timer can report when a timeout is in the past, so
238 set_next_event is capable of returning -ETIME when appropriate.
239 This interface is used when available.
244 Get a hypervisor absolute time. In theory we could maintain an
245 offset between the kernel's time and the hypervisor's time, and
246 apply that to a kernel's absolute timeout. Unfortunately the
247 hypervisor and kernel times can drift even if the kernel is using
248 the Xen clocksource, because ntp can warp the kernel's clocksource.
250 static s64
get_abs_timeout(unsigned long delta
)
252 return xen_clocksource_read() + delta
;
255 static void xen_timerop_set_mode(enum clock_event_mode mode
,
256 struct clock_event_device
*evt
)
259 case CLOCK_EVT_MODE_PERIODIC
:
264 case CLOCK_EVT_MODE_ONESHOT
:
265 case CLOCK_EVT_MODE_RESUME
:
268 case CLOCK_EVT_MODE_UNUSED
:
269 case CLOCK_EVT_MODE_SHUTDOWN
:
270 HYPERVISOR_set_timer_op(0); /* cancel timeout */
275 static int xen_timerop_set_next_event(unsigned long delta
,
276 struct clock_event_device
*evt
)
278 WARN_ON(evt
->mode
!= CLOCK_EVT_MODE_ONESHOT
);
280 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta
)) < 0)
283 /* We may have missed the deadline, but there's no real way of
284 knowing for sure. If the event was in the past, then we'll
285 get an immediate interrupt. */
290 static const struct clock_event_device xen_timerop_clockevent
= {
292 .features
= CLOCK_EVT_FEAT_ONESHOT
,
294 .max_delta_ns
= 0xffffffff,
295 .min_delta_ns
= TIMER_SLOP
,
301 .set_mode
= xen_timerop_set_mode
,
302 .set_next_event
= xen_timerop_set_next_event
,
307 static void xen_vcpuop_set_mode(enum clock_event_mode mode
,
308 struct clock_event_device
*evt
)
310 int cpu
= smp_processor_id();
313 case CLOCK_EVT_MODE_PERIODIC
:
314 WARN_ON(1); /* unsupported */
317 case CLOCK_EVT_MODE_ONESHOT
:
318 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer
, cpu
, NULL
))
322 case CLOCK_EVT_MODE_UNUSED
:
323 case CLOCK_EVT_MODE_SHUTDOWN
:
324 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer
, cpu
, NULL
) ||
325 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer
, cpu
, NULL
))
328 case CLOCK_EVT_MODE_RESUME
:
333 static int xen_vcpuop_set_next_event(unsigned long delta
,
334 struct clock_event_device
*evt
)
336 int cpu
= smp_processor_id();
337 struct vcpu_set_singleshot_timer single
;
340 WARN_ON(evt
->mode
!= CLOCK_EVT_MODE_ONESHOT
);
342 single
.timeout_abs_ns
= get_abs_timeout(delta
);
343 single
.flags
= VCPU_SSHOTTMR_future
;
345 ret
= HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer
, cpu
, &single
);
347 BUG_ON(ret
!= 0 && ret
!= -ETIME
);
352 static const struct clock_event_device xen_vcpuop_clockevent
= {
354 .features
= CLOCK_EVT_FEAT_ONESHOT
,
356 .max_delta_ns
= 0xffffffff,
357 .min_delta_ns
= TIMER_SLOP
,
363 .set_mode
= xen_vcpuop_set_mode
,
364 .set_next_event
= xen_vcpuop_set_next_event
,
367 static const struct clock_event_device
*xen_clockevent
=
368 &xen_timerop_clockevent
;
369 static DEFINE_PER_CPU(struct clock_event_device
, xen_clock_events
);
371 static irqreturn_t
xen_timer_interrupt(int irq
, void *dev_id
)
373 struct clock_event_device
*evt
= &__get_cpu_var(xen_clock_events
);
377 if (evt
->event_handler
) {
378 evt
->event_handler(evt
);
382 do_stolen_accounting();
387 void xen_setup_timer(int cpu
)
390 struct clock_event_device
*evt
;
393 printk(KERN_INFO
"installing Xen timer for CPU %d\n", cpu
);
395 name
= kasprintf(GFP_KERNEL
, "timer%d", cpu
);
397 name
= "<timer kasprintf failed>";
399 irq
= bind_virq_to_irqhandler(VIRQ_TIMER
, cpu
, xen_timer_interrupt
,
400 IRQF_DISABLED
|IRQF_PERCPU
|IRQF_NOBALANCING
|IRQF_TIMER
,
403 evt
= &per_cpu(xen_clock_events
, cpu
);
404 memcpy(evt
, xen_clockevent
, sizeof(*evt
));
406 evt
->cpumask
= cpumask_of(cpu
);
410 void xen_teardown_timer(int cpu
)
412 struct clock_event_device
*evt
;
414 evt
= &per_cpu(xen_clock_events
, cpu
);
415 unbind_from_irqhandler(evt
->irq
, NULL
);
418 void xen_setup_cpu_clockevents(void)
420 BUG_ON(preemptible());
422 clockevents_register_device(&__get_cpu_var(xen_clock_events
));
425 void xen_timer_resume(void)
429 if (xen_clockevent
!= &xen_vcpuop_clockevent
)
432 for_each_online_cpu(cpu
) {
433 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer
, cpu
, NULL
))
438 static const struct pv_time_ops xen_time_ops __initdata
= {
439 .sched_clock
= xen_clocksource_read
,
442 static __init
void xen_time_init(void)
444 int cpu
= smp_processor_id();
447 clocksource_register(&xen_clocksource
);
449 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer
, cpu
, NULL
) == 0) {
450 /* Successfully turned off 100Hz tick, so we have the
451 vcpuop-based timer interface */
452 printk(KERN_DEBUG
"Xen: using vcpuop timer interface\n");
453 xen_clockevent
= &xen_vcpuop_clockevent
;
456 /* Set initial system time with full resolution */
457 xen_read_wallclock(&tp
);
458 do_settimeofday(&tp
);
460 setup_force_cpu_cap(X86_FEATURE_TSC
);
462 xen_setup_runstate_info(cpu
);
463 xen_setup_timer(cpu
);
464 xen_setup_cpu_clockevents();
467 __init
void xen_init_time_ops(void)
469 pv_time_ops
= xen_time_ops
;
471 x86_init
.timers
.timer_init
= xen_time_init
;
472 x86_init
.timers
.setup_percpu_clockev
= x86_init_noop
;
473 x86_cpuinit
.setup_percpu_clockev
= x86_init_noop
;
475 x86_platform
.calibrate_tsc
= xen_tsc_khz
;
476 x86_platform
.get_wallclock
= xen_get_wallclock
;
477 x86_platform
.set_wallclock
= xen_set_wallclock
;
480 #ifdef CONFIG_XEN_PVHVM
481 static void xen_hvm_setup_cpu_clockevents(void)
483 int cpu
= smp_processor_id();
484 xen_setup_runstate_info(cpu
);
485 xen_setup_timer(cpu
);
486 xen_setup_cpu_clockevents();
489 __init
void xen_hvm_init_time_ops(void)
491 /* vector callback is needed otherwise we cannot receive interrupts
492 * on cpu > 0 and at this point we don't know how many cpus are
494 if (!xen_have_vector_callback
)
496 if (!xen_feature(XENFEAT_hvm_safe_pvclock
)) {
497 printk(KERN_INFO
"Xen doesn't support pvclock on HVM,"
498 "disable pv timer\n");
502 pv_time_ops
= xen_time_ops
;
503 x86_init
.timers
.setup_percpu_clockev
= xen_time_init
;
504 x86_cpuinit
.setup_percpu_clockev
= xen_hvm_setup_cpu_clockevents
;
506 x86_platform
.calibrate_tsc
= xen_tsc_khz
;
507 x86_platform
.get_wallclock
= xen_get_wallclock
;
508 x86_platform
.set_wallclock
= xen_set_wallclock
;