2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/time.h>
13 #include <linux/delay.h>
14 #include <linux/init.h>
15 #include <linux/smp.h>
16 #include <linux/types.h>
17 #include <linux/timex.h>
18 #include <linux/notifier.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/rcupdate.h>
21 #include <linux/posix-timers.h>
22 #include <linux/cpu.h>
24 #include <asm/s390_ext.h>
25 #include <asm/timer.h>
26 #include <asm/irq_regs.h>
27 #include <asm/cputime.h>
29 static DEFINE_PER_CPU(struct vtimer_queue
, virt_cpu_timer
);
31 DEFINE_PER_CPU(struct s390_idle_data
, s390_idle
);
33 static inline __u64
get_vtimer(void)
37 asm volatile("STPT %0" : "=m" (timer
));
41 static inline void set_vtimer(__u64 expires
)
45 asm volatile (" STPT %0\n" /* Store current cpu timer value */
46 " SPT %1" /* Set new value immediatly afterwards */
47 : "=m" (timer
) : "m" (expires
) );
48 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- timer
;
49 S390_lowcore
.last_update_timer
= expires
;
53 * Update process times based on virtual cpu times stored by entry.S
54 * to the lowcore fields user_timer, system_timer & steal_clock.
56 static void do_account_vtime(struct task_struct
*tsk
, int hardirq_offset
)
58 struct thread_info
*ti
= task_thread_info(tsk
);
59 __u64 timer
, clock
, user
, system
, steal
;
61 timer
= S390_lowcore
.last_update_timer
;
62 clock
= S390_lowcore
.last_update_clock
;
63 asm volatile (" STPT %0\n" /* Store current cpu timer value */
64 " STCK %1" /* Store current tod clock value */
65 : "=m" (S390_lowcore
.last_update_timer
),
66 "=m" (S390_lowcore
.last_update_clock
) );
67 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
68 S390_lowcore
.steal_timer
+= S390_lowcore
.last_update_clock
- clock
;
70 user
= S390_lowcore
.user_timer
- ti
->user_timer
;
71 S390_lowcore
.steal_timer
-= user
;
72 ti
->user_timer
= S390_lowcore
.user_timer
;
73 account_user_time(tsk
, user
, user
);
75 system
= S390_lowcore
.system_timer
- ti
->system_timer
;
76 S390_lowcore
.steal_timer
-= system
;
77 ti
->system_timer
= S390_lowcore
.system_timer
;
78 account_system_time(tsk
, hardirq_offset
, system
, system
);
80 steal
= S390_lowcore
.steal_timer
;
81 if ((s64
) steal
> 0) {
82 S390_lowcore
.steal_timer
= 0;
83 account_steal_time(steal
);
87 void account_vtime(struct task_struct
*prev
, struct task_struct
*next
)
89 struct thread_info
*ti
;
91 do_account_vtime(prev
, 0);
92 ti
= task_thread_info(prev
);
93 ti
->user_timer
= S390_lowcore
.user_timer
;
94 ti
->system_timer
= S390_lowcore
.system_timer
;
95 ti
= task_thread_info(next
);
96 S390_lowcore
.user_timer
= ti
->user_timer
;
97 S390_lowcore
.system_timer
= ti
->system_timer
;
100 void account_process_tick(struct task_struct
*tsk
, int user_tick
)
102 do_account_vtime(tsk
, HARDIRQ_OFFSET
);
106 * Update process times based on virtual cpu times stored by entry.S
107 * to the lowcore fields user_timer, system_timer & steal_clock.
109 void account_system_vtime(struct task_struct
*tsk
)
111 struct thread_info
*ti
= task_thread_info(tsk
);
114 timer
= S390_lowcore
.last_update_timer
;
115 S390_lowcore
.last_update_timer
= get_vtimer();
116 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
118 system
= S390_lowcore
.system_timer
- ti
->system_timer
;
119 S390_lowcore
.steal_timer
-= system
;
120 ti
->system_timer
= S390_lowcore
.system_timer
;
121 account_system_time(tsk
, 0, system
, system
);
123 EXPORT_SYMBOL_GPL(account_system_vtime
);
125 void vtime_start_cpu(__u64 int_clock
, __u64 enter_timer
)
127 struct s390_idle_data
*idle
= &__get_cpu_var(s390_idle
);
128 struct vtimer_queue
*vq
= &__get_cpu_var(virt_cpu_timer
);
129 __u64 idle_time
, expires
;
131 if (idle
->idle_enter
== 0ULL)
134 /* Account time spent with enabled wait psw loaded as idle time. */
135 idle_time
= int_clock
- idle
->idle_enter
;
136 account_idle_time(idle_time
);
137 S390_lowcore
.steal_timer
+=
138 idle
->idle_enter
- S390_lowcore
.last_update_clock
;
139 S390_lowcore
.last_update_clock
= int_clock
;
141 /* Account system time spent going idle. */
142 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- vq
->idle
;
143 S390_lowcore
.last_update_timer
= enter_timer
;
145 /* Restart vtime CPU timer */
147 /* Program old expire value but first save progress. */
148 expires
= vq
->idle
- enter_timer
;
149 expires
+= get_vtimer();
152 /* Don't account the CPU timer delta while the cpu was idle. */
153 vq
->elapsed
-= vq
->idle
- enter_timer
;
158 idle
->idle_time
+= idle_time
;
159 idle
->idle_enter
= 0ULL;
165 void vtime_stop_cpu(void)
167 struct s390_idle_data
*idle
= &__get_cpu_var(s390_idle
);
168 struct vtimer_queue
*vq
= &__get_cpu_var(virt_cpu_timer
);
171 /* Wait for external, I/O or machine check interrupt. */
172 psw
.mask
= psw_kernel_bits
| PSW_MASK_WAIT
| PSW_MASK_IO
| PSW_MASK_EXT
;
174 idle
->nohz_delay
= 0;
176 /* Check if the CPU timer needs to be reprogrammed. */
178 __u64 vmax
= VTIMER_MAX_SLICE
;
180 * The inline assembly is equivalent to
181 * vq->idle = get_cpu_timer();
182 * set_cpu_timer(VTIMER_MAX_SLICE);
183 * idle->idle_enter = get_clock();
184 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
185 * PSW_MASK_IO | PSW_MASK_EXT);
186 * The difference is that the inline assembly makes sure that
187 * the last three instruction are stpt, stck and lpsw in that
188 * order. This is done to increase the precision.
195 #else /* CONFIG_64BIT */
198 #endif /* CONFIG_64BIT */
204 #else /* CONFIG_64BIT */
206 #endif /* CONFIG_64BIT */
208 : "=m" (idle
->idle_enter
), "=m" (vq
->idle
)
209 : "a" (&psw
), "a" (&idle
->idle_enter
),
210 "a" (&vq
->idle
), "a" (&vmax
), "m" (vmax
), "m" (psw
)
211 : "memory", "cc", "1");
214 * The inline assembly is equivalent to
215 * vq->idle = get_cpu_timer();
216 * idle->idle_enter = get_clock();
217 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
218 * PSW_MASK_IO | PSW_MASK_EXT);
219 * The difference is that the inline assembly makes sure that
220 * the last three instruction are stpt, stck and lpsw in that
221 * order. This is done to increase the precision.
228 #else /* CONFIG_64BIT */
231 #endif /* CONFIG_64BIT */
236 #else /* CONFIG_64BIT */
238 #endif /* CONFIG_64BIT */
240 : "=m" (idle
->idle_enter
), "=m" (vq
->idle
)
241 : "a" (&psw
), "a" (&idle
->idle_enter
),
242 "a" (&vq
->idle
), "m" (psw
)
243 : "memory", "cc", "1");
247 cputime64_t
s390_get_idle_time(int cpu
)
249 struct s390_idle_data
*idle
;
250 unsigned long long now
, idle_time
, idle_enter
;
251 unsigned int sequence
;
253 idle
= &per_cpu(s390_idle
, cpu
);
257 sequence
= idle
->sequence
;
262 idle_enter
= idle
->idle_enter
;
263 if (idle_enter
!= 0ULL && idle_enter
< now
)
264 idle_time
= now
- idle_enter
;
266 if (idle
->sequence
!= sequence
)
272 * Sorted add to a list. List is linear searched until first bigger
275 static void list_add_sorted(struct vtimer_list
*timer
, struct list_head
*head
)
277 struct vtimer_list
*event
;
279 list_for_each_entry(event
, head
, entry
) {
280 if (event
->expires
> timer
->expires
) {
281 list_add_tail(&timer
->entry
, &event
->entry
);
285 list_add_tail(&timer
->entry
, head
);
289 * Do the callback functions of expired vtimer events.
290 * Called from within the interrupt handler.
292 static void do_callbacks(struct list_head
*cb_list
)
294 struct vtimer_queue
*vq
;
295 struct vtimer_list
*event
, *tmp
;
297 if (list_empty(cb_list
))
300 vq
= &__get_cpu_var(virt_cpu_timer
);
302 list_for_each_entry_safe(event
, tmp
, cb_list
, entry
) {
303 list_del_init(&event
->entry
);
304 (event
->function
)(event
->data
);
305 if (event
->interval
) {
306 /* Recharge interval timer */
307 event
->expires
= event
->interval
+ vq
->elapsed
;
308 spin_lock(&vq
->lock
);
309 list_add_sorted(event
, &vq
->list
);
310 spin_unlock(&vq
->lock
);
316 * Handler for the virtual CPU timer.
318 static void do_cpu_timer_interrupt(__u16 error_code
)
320 struct vtimer_queue
*vq
;
321 struct vtimer_list
*event
, *tmp
;
322 struct list_head cb_list
; /* the callback queue */
325 INIT_LIST_HEAD(&cb_list
);
326 vq
= &__get_cpu_var(virt_cpu_timer
);
328 /* walk timer list, fire all expired events */
329 spin_lock(&vq
->lock
);
331 elapsed
= vq
->elapsed
+ (vq
->timer
- S390_lowcore
.async_enter_timer
);
332 BUG_ON((s64
) elapsed
< 0);
334 list_for_each_entry_safe(event
, tmp
, &vq
->list
, entry
) {
335 if (event
->expires
< elapsed
)
336 /* move expired timer to the callback queue */
337 list_move_tail(&event
->entry
, &cb_list
);
339 event
->expires
-= elapsed
;
341 spin_unlock(&vq
->lock
);
343 vq
->do_spt
= list_empty(&cb_list
);
344 do_callbacks(&cb_list
);
346 /* next event is first in list */
347 next
= VTIMER_MAX_SLICE
;
348 spin_lock(&vq
->lock
);
349 if (!list_empty(&vq
->list
)) {
350 event
= list_first_entry(&vq
->list
, struct vtimer_list
, entry
);
351 next
= event
->expires
;
354 spin_unlock(&vq
->lock
);
356 * To improve precision add the time spent by the
357 * interrupt handler to the elapsed time.
358 * Note: CPU timer counts down and we got an interrupt,
359 * the current content is negative
361 elapsed
= S390_lowcore
.async_enter_timer
- get_vtimer();
362 set_vtimer(next
- elapsed
);
363 vq
->timer
= next
- elapsed
;
364 vq
->elapsed
= elapsed
;
367 void init_virt_timer(struct vtimer_list
*timer
)
369 timer
->function
= NULL
;
370 INIT_LIST_HEAD(&timer
->entry
);
372 EXPORT_SYMBOL(init_virt_timer
);
374 static inline int vtimer_pending(struct vtimer_list
*timer
)
376 return (!list_empty(&timer
->entry
));
380 * this function should only run on the specified CPU
382 static void internal_add_vtimer(struct vtimer_list
*timer
)
384 struct vtimer_queue
*vq
;
388 vq
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
389 spin_lock_irqsave(&vq
->lock
, flags
);
391 BUG_ON(timer
->cpu
!= smp_processor_id());
393 if (list_empty(&vq
->list
)) {
394 /* First timer on this cpu, just program it. */
395 list_add(&timer
->entry
, &vq
->list
);
396 set_vtimer(timer
->expires
);
397 vq
->timer
= timer
->expires
;
400 /* Check progress of old timers. */
401 expires
= timer
->expires
;
403 if (likely((s64
) expires
< (s64
) left
)) {
404 /* The new timer expires before the current timer. */
406 vq
->elapsed
+= vq
->timer
- left
;
409 vq
->elapsed
+= vq
->timer
- left
;
412 /* Insert new timer into per cpu list. */
413 timer
->expires
+= vq
->elapsed
;
414 list_add_sorted(timer
, &vq
->list
);
417 spin_unlock_irqrestore(&vq
->lock
, flags
);
418 /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
422 static inline void prepare_vtimer(struct vtimer_list
*timer
)
424 BUG_ON(!timer
->function
);
425 BUG_ON(!timer
->expires
|| timer
->expires
> VTIMER_MAX_SLICE
);
426 BUG_ON(vtimer_pending(timer
));
427 timer
->cpu
= get_cpu();
431 * add_virt_timer - add an oneshot virtual CPU timer
433 void add_virt_timer(void *new)
435 struct vtimer_list
*timer
;
437 timer
= (struct vtimer_list
*)new;
438 prepare_vtimer(timer
);
440 internal_add_vtimer(timer
);
442 EXPORT_SYMBOL(add_virt_timer
);
445 * add_virt_timer_int - add an interval virtual CPU timer
447 void add_virt_timer_periodic(void *new)
449 struct vtimer_list
*timer
;
451 timer
= (struct vtimer_list
*)new;
452 prepare_vtimer(timer
);
453 timer
->interval
= timer
->expires
;
454 internal_add_vtimer(timer
);
456 EXPORT_SYMBOL(add_virt_timer_periodic
);
458 int __mod_vtimer(struct vtimer_list
*timer
, __u64 expires
, int periodic
)
460 struct vtimer_queue
*vq
;
464 BUG_ON(!timer
->function
);
465 BUG_ON(!expires
|| expires
> VTIMER_MAX_SLICE
);
467 if (timer
->expires
== expires
&& vtimer_pending(timer
))
471 vq
= &per_cpu(virt_cpu_timer
, cpu
);
473 /* disable interrupts before test if timer is pending */
474 spin_lock_irqsave(&vq
->lock
, flags
);
476 /* if timer isn't pending add it on the current CPU */
477 if (!vtimer_pending(timer
)) {
478 spin_unlock_irqrestore(&vq
->lock
, flags
);
481 timer
->interval
= expires
;
484 timer
->expires
= expires
;
486 internal_add_vtimer(timer
);
490 /* check if we run on the right CPU */
491 BUG_ON(timer
->cpu
!= cpu
);
493 list_del_init(&timer
->entry
);
494 timer
->expires
= expires
;
496 timer
->interval
= expires
;
498 /* the timer can't expire anymore so we can release the lock */
499 spin_unlock_irqrestore(&vq
->lock
, flags
);
500 internal_add_vtimer(timer
);
505 * If we change a pending timer the function must be called on the CPU
506 * where the timer is running on.
508 * returns whether it has modified a pending timer (1) or not (0)
510 int mod_virt_timer(struct vtimer_list
*timer
, __u64 expires
)
512 return __mod_vtimer(timer
, expires
, 0);
514 EXPORT_SYMBOL(mod_virt_timer
);
517 * If we change a pending timer the function must be called on the CPU
518 * where the timer is running on.
520 * returns whether it has modified a pending timer (1) or not (0)
522 int mod_virt_timer_periodic(struct vtimer_list
*timer
, __u64 expires
)
524 return __mod_vtimer(timer
, expires
, 1);
526 EXPORT_SYMBOL(mod_virt_timer_periodic
);
529 * delete a virtual timer
531 * returns whether the deleted timer was pending (1) or not (0)
533 int del_virt_timer(struct vtimer_list
*timer
)
536 struct vtimer_queue
*vq
;
538 /* check if timer is pending */
539 if (!vtimer_pending(timer
))
542 vq
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
543 spin_lock_irqsave(&vq
->lock
, flags
);
545 /* we don't interrupt a running timer, just let it expire! */
546 list_del_init(&timer
->entry
);
548 spin_unlock_irqrestore(&vq
->lock
, flags
);
551 EXPORT_SYMBOL(del_virt_timer
);
554 * Start the virtual CPU timer on the current CPU.
556 void init_cpu_vtimer(void)
558 struct vtimer_queue
*vq
;
560 /* initialize per cpu vtimer structure */
561 vq
= &__get_cpu_var(virt_cpu_timer
);
562 INIT_LIST_HEAD(&vq
->list
);
563 spin_lock_init(&vq
->lock
);
565 /* enable cpu timer interrupts */
569 static int __cpuinit
s390_nohz_notify(struct notifier_block
*self
,
570 unsigned long action
, void *hcpu
)
572 struct s390_idle_data
*idle
;
573 long cpu
= (long) hcpu
;
575 idle
= &per_cpu(s390_idle
, cpu
);
578 case CPU_DYING_FROZEN
:
579 idle
->nohz_delay
= 0;
586 void __init
vtime_init(void)
588 /* request the cpu timer external interrupt */
589 if (register_external_interrupt(0x1005, do_cpu_timer_interrupt
))
590 panic("Couldn't request external interrupt 0x1005");
592 /* Enable cpu timer interrupts on the boot cpu. */
594 cpu_notifier(s390_nohz_notify
, 0);