2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/time.h>
13 #include <linux/delay.h>
14 #include <linux/init.h>
15 #include <linux/smp.h>
16 #include <linux/types.h>
17 #include <linux/timex.h>
18 #include <linux/notifier.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/rcupdate.h>
21 #include <linux/posix-timers.h>
23 #include <asm/s390_ext.h>
24 #include <asm/timer.h>
25 #include <asm/irq_regs.h>
28 static ext_int_info_t ext_int_info_timer
;
30 static DEFINE_PER_CPU(struct vtimer_queue
, virt_cpu_timer
);
32 DEFINE_PER_CPU(struct s390_idle_data
, s390_idle
) = {
33 .lock
= __SPIN_LOCK_UNLOCKED(s390_idle
.lock
)
36 static inline __u64
get_vtimer(void)
40 asm volatile("STPT %0" : "=m" (timer
));
44 static inline void set_vtimer(__u64 expires
)
48 asm volatile (" STPT %0\n" /* Store current cpu timer value */
49 " SPT %1" /* Set new value immediatly afterwards */
50 : "=m" (timer
) : "m" (expires
) );
51 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- timer
;
52 S390_lowcore
.last_update_timer
= expires
;
56 * Update process times based on virtual cpu times stored by entry.S
57 * to the lowcore fields user_timer, system_timer & steal_clock.
59 static void do_account_vtime(struct task_struct
*tsk
, int hardirq_offset
)
61 struct thread_info
*ti
= task_thread_info(tsk
);
62 __u64 timer
, clock
, user
, system
, steal
;
64 timer
= S390_lowcore
.last_update_timer
;
65 clock
= S390_lowcore
.last_update_clock
;
66 asm volatile (" STPT %0\n" /* Store current cpu timer value */
67 " STCK %1" /* Store current tod clock value */
68 : "=m" (S390_lowcore
.last_update_timer
),
69 "=m" (S390_lowcore
.last_update_clock
) );
70 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
71 S390_lowcore
.steal_timer
+= S390_lowcore
.last_update_clock
- clock
;
73 user
= S390_lowcore
.user_timer
- ti
->user_timer
;
74 S390_lowcore
.steal_timer
-= user
;
75 ti
->user_timer
= S390_lowcore
.user_timer
;
76 account_user_time(tsk
, user
, user
);
78 system
= S390_lowcore
.system_timer
- ti
->system_timer
;
79 S390_lowcore
.steal_timer
-= system
;
80 ti
->system_timer
= S390_lowcore
.system_timer
;
81 account_system_time(tsk
, hardirq_offset
, system
, system
);
83 steal
= S390_lowcore
.steal_timer
;
84 if ((s64
) steal
> 0) {
85 S390_lowcore
.steal_timer
= 0;
86 account_steal_time(steal
);
90 void account_vtime(struct task_struct
*prev
, struct task_struct
*next
)
92 struct thread_info
*ti
;
94 do_account_vtime(prev
, 0);
95 ti
= task_thread_info(prev
);
96 ti
->user_timer
= S390_lowcore
.user_timer
;
97 ti
->system_timer
= S390_lowcore
.system_timer
;
98 ti
= task_thread_info(next
);
99 S390_lowcore
.user_timer
= ti
->user_timer
;
100 S390_lowcore
.system_timer
= ti
->system_timer
;
103 void account_process_tick(struct task_struct
*tsk
, int user_tick
)
105 do_account_vtime(tsk
, HARDIRQ_OFFSET
);
109 * Update process times based on virtual cpu times stored by entry.S
110 * to the lowcore fields user_timer, system_timer & steal_clock.
112 void account_system_vtime(struct task_struct
*tsk
)
114 struct thread_info
*ti
= task_thread_info(tsk
);
117 timer
= S390_lowcore
.last_update_timer
;
118 S390_lowcore
.last_update_timer
= get_vtimer();
119 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
121 system
= S390_lowcore
.system_timer
- ti
->system_timer
;
122 S390_lowcore
.steal_timer
-= system
;
123 ti
->system_timer
= S390_lowcore
.system_timer
;
124 account_system_time(tsk
, 0, system
, system
);
126 EXPORT_SYMBOL_GPL(account_system_vtime
);
128 void vtime_start_cpu(void)
130 struct s390_idle_data
*idle
= &__get_cpu_var(s390_idle
);
131 struct vtimer_queue
*vq
= &__get_cpu_var(virt_cpu_timer
);
132 __u64 idle_time
, expires
;
134 /* Account time spent with enabled wait psw loaded as idle time. */
135 idle_time
= S390_lowcore
.int_clock
- idle
->idle_enter
;
136 account_idle_time(idle_time
);
137 S390_lowcore
.last_update_clock
= S390_lowcore
.int_clock
;
139 /* Account system time spent going idle. */
140 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- vq
->idle
;
141 S390_lowcore
.last_update_timer
= S390_lowcore
.async_enter_timer
;
143 /* Restart vtime CPU timer */
145 /* Program old expire value but first save progress. */
146 expires
= vq
->idle
- S390_lowcore
.async_enter_timer
;
147 expires
+= get_vtimer();
150 /* Don't account the CPU timer delta while the cpu was idle. */
151 vq
->elapsed
-= vq
->idle
- S390_lowcore
.async_enter_timer
;
154 spin_lock(&idle
->lock
);
155 idle
->idle_time
+= idle_time
;
156 idle
->idle_enter
= 0ULL;
158 spin_unlock(&idle
->lock
);
161 void vtime_stop_cpu(void)
163 struct s390_idle_data
*idle
= &__get_cpu_var(s390_idle
);
164 struct vtimer_queue
*vq
= &__get_cpu_var(virt_cpu_timer
);
167 /* Wait for external, I/O or machine check interrupt. */
168 psw
.mask
= psw_kernel_bits
| PSW_MASK_WAIT
| PSW_MASK_IO
| PSW_MASK_EXT
;
170 /* Check if the CPU timer needs to be reprogrammed. */
172 __u64 vmax
= VTIMER_MAX_SLICE
;
174 * The inline assembly is equivalent to
175 * vq->idle = get_cpu_timer();
176 * set_cpu_timer(VTIMER_MAX_SLICE);
177 * idle->idle_enter = get_clock();
178 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
179 * PSW_MASK_IO | PSW_MASK_EXT);
180 * The difference is that the inline assembly makes sure that
181 * the last three instruction are stpt, stck and lpsw in that
182 * order. This is done to increase the precision.
189 #else /* CONFIG_64BIT */
192 #endif /* CONFIG_64BIT */
198 #else /* CONFIG_64BIT */
200 #endif /* CONFIG_64BIT */
202 : "=m" (idle
->idle_enter
), "=m" (vq
->idle
)
203 : "a" (&psw
), "a" (&idle
->idle_enter
),
204 "a" (&vq
->idle
), "a" (&vmax
), "m" (vmax
), "m" (psw
)
205 : "memory", "cc", "1");
208 * The inline assembly is equivalent to
209 * vq->idle = get_cpu_timer();
210 * idle->idle_enter = get_clock();
211 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
212 * PSW_MASK_IO | PSW_MASK_EXT);
213 * The difference is that the inline assembly makes sure that
214 * the last three instruction are stpt, stck and lpsw in that
215 * order. This is done to increase the precision.
222 #else /* CONFIG_64BIT */
225 #endif /* CONFIG_64BIT */
230 #else /* CONFIG_64BIT */
232 #endif /* CONFIG_64BIT */
234 : "=m" (idle
->idle_enter
), "=m" (vq
->idle
)
235 : "a" (&psw
), "a" (&idle
->idle_enter
),
236 "a" (&vq
->idle
), "m" (psw
)
237 : "memory", "cc", "1");
242 * Sorted add to a list. List is linear searched until first bigger
245 static void list_add_sorted(struct vtimer_list
*timer
, struct list_head
*head
)
247 struct vtimer_list
*event
;
249 list_for_each_entry(event
, head
, entry
) {
250 if (event
->expires
> timer
->expires
) {
251 list_add_tail(&timer
->entry
, &event
->entry
);
255 list_add_tail(&timer
->entry
, head
);
259 * Do the callback functions of expired vtimer events.
260 * Called from within the interrupt handler.
262 static void do_callbacks(struct list_head
*cb_list
)
264 struct vtimer_queue
*vq
;
265 struct vtimer_list
*event
, *tmp
;
267 if (list_empty(cb_list
))
270 vq
= &__get_cpu_var(virt_cpu_timer
);
272 list_for_each_entry_safe(event
, tmp
, cb_list
, entry
) {
273 list_del_init(&event
->entry
);
274 (event
->function
)(event
->data
);
275 if (event
->interval
) {
276 /* Recharge interval timer */
277 event
->expires
= event
->interval
+ vq
->elapsed
;
278 spin_lock(&vq
->lock
);
279 list_add_sorted(event
, &vq
->list
);
280 spin_unlock(&vq
->lock
);
286 * Handler for the virtual CPU timer.
288 static void do_cpu_timer_interrupt(__u16 error_code
)
290 struct vtimer_queue
*vq
;
291 struct vtimer_list
*event
, *tmp
;
292 struct list_head cb_list
; /* the callback queue */
295 INIT_LIST_HEAD(&cb_list
);
296 vq
= &__get_cpu_var(virt_cpu_timer
);
298 /* walk timer list, fire all expired events */
299 spin_lock(&vq
->lock
);
301 elapsed
= vq
->elapsed
+ (vq
->timer
- S390_lowcore
.async_enter_timer
);
302 BUG_ON((s64
) elapsed
< 0);
304 list_for_each_entry_safe(event
, tmp
, &vq
->list
, entry
) {
305 if (event
->expires
< elapsed
)
306 /* move expired timer to the callback queue */
307 list_move_tail(&event
->entry
, &cb_list
);
309 event
->expires
-= elapsed
;
311 spin_unlock(&vq
->lock
);
313 vq
->do_spt
= list_empty(&cb_list
);
314 do_callbacks(&cb_list
);
316 /* next event is first in list */
317 next
= VTIMER_MAX_SLICE
;
318 spin_lock(&vq
->lock
);
319 if (!list_empty(&vq
->list
)) {
320 event
= list_first_entry(&vq
->list
, struct vtimer_list
, entry
);
321 next
= event
->expires
;
324 spin_unlock(&vq
->lock
);
326 * To improve precision add the time spent by the
327 * interrupt handler to the elapsed time.
328 * Note: CPU timer counts down and we got an interrupt,
329 * the current content is negative
331 elapsed
= S390_lowcore
.async_enter_timer
- get_vtimer();
332 set_vtimer(next
- elapsed
);
333 vq
->timer
= next
- elapsed
;
334 vq
->elapsed
= elapsed
;
337 void init_virt_timer(struct vtimer_list
*timer
)
339 timer
->function
= NULL
;
340 INIT_LIST_HEAD(&timer
->entry
);
342 EXPORT_SYMBOL(init_virt_timer
);
344 static inline int vtimer_pending(struct vtimer_list
*timer
)
346 return (!list_empty(&timer
->entry
));
350 * this function should only run on the specified CPU
352 static void internal_add_vtimer(struct vtimer_list
*timer
)
354 struct vtimer_queue
*vq
;
358 vq
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
359 spin_lock_irqsave(&vq
->lock
, flags
);
361 BUG_ON(timer
->cpu
!= smp_processor_id());
363 if (list_empty(&vq
->list
)) {
364 /* First timer on this cpu, just program it. */
365 list_add(&timer
->entry
, &vq
->list
);
366 set_vtimer(timer
->expires
);
367 vq
->timer
= timer
->expires
;
370 /* Check progress of old timers. */
371 expires
= timer
->expires
;
373 if (likely((s64
) expires
< (s64
) left
)) {
374 /* The new timer expires before the current timer. */
376 vq
->elapsed
+= vq
->timer
- left
;
379 vq
->elapsed
+= vq
->timer
- left
;
382 /* Insert new timer into per cpu list. */
383 timer
->expires
+= vq
->elapsed
;
384 list_add_sorted(timer
, &vq
->list
);
387 spin_unlock_irqrestore(&vq
->lock
, flags
);
388 /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
392 static inline void prepare_vtimer(struct vtimer_list
*timer
)
394 BUG_ON(!timer
->function
);
395 BUG_ON(!timer
->expires
|| timer
->expires
> VTIMER_MAX_SLICE
);
396 BUG_ON(vtimer_pending(timer
));
397 timer
->cpu
= get_cpu();
401 * add_virt_timer - add an oneshot virtual CPU timer
403 void add_virt_timer(void *new)
405 struct vtimer_list
*timer
;
407 timer
= (struct vtimer_list
*)new;
408 prepare_vtimer(timer
);
410 internal_add_vtimer(timer
);
412 EXPORT_SYMBOL(add_virt_timer
);
415 * add_virt_timer_int - add an interval virtual CPU timer
417 void add_virt_timer_periodic(void *new)
419 struct vtimer_list
*timer
;
421 timer
= (struct vtimer_list
*)new;
422 prepare_vtimer(timer
);
423 timer
->interval
= timer
->expires
;
424 internal_add_vtimer(timer
);
426 EXPORT_SYMBOL(add_virt_timer_periodic
);
429 * If we change a pending timer the function must be called on the CPU
430 * where the timer is running on, e.g. by smp_call_function_single()
432 * The original mod_timer adds the timer if it is not pending. For
433 * compatibility we do the same. The timer will be added on the current
434 * CPU as a oneshot timer.
436 * returns whether it has modified a pending timer (1) or not (0)
438 int mod_virt_timer(struct vtimer_list
*timer
, __u64 expires
)
440 struct vtimer_queue
*vq
;
444 BUG_ON(!timer
->function
);
445 BUG_ON(!expires
|| expires
> VTIMER_MAX_SLICE
);
448 * This is a common optimization triggered by the
449 * networking code - if the timer is re-modified
450 * to be the same thing then just return:
452 if (timer
->expires
== expires
&& vtimer_pending(timer
))
456 vq
= &per_cpu(virt_cpu_timer
, cpu
);
458 /* check if we run on the right CPU */
459 BUG_ON(timer
->cpu
!= cpu
);
461 /* disable interrupts before test if timer is pending */
462 spin_lock_irqsave(&vq
->lock
, flags
);
464 /* if timer isn't pending add it on the current CPU */
465 if (!vtimer_pending(timer
)) {
466 spin_unlock_irqrestore(&vq
->lock
, flags
);
467 /* we do not activate an interval timer with mod_virt_timer */
469 timer
->expires
= expires
;
471 internal_add_vtimer(timer
);
475 list_del_init(&timer
->entry
);
476 timer
->expires
= expires
;
478 /* also change the interval if we have an interval timer */
480 timer
->interval
= expires
;
482 /* the timer can't expire anymore so we can release the lock */
483 spin_unlock_irqrestore(&vq
->lock
, flags
);
484 internal_add_vtimer(timer
);
487 EXPORT_SYMBOL(mod_virt_timer
);
490 * delete a virtual timer
492 * returns whether the deleted timer was pending (1) or not (0)
494 int del_virt_timer(struct vtimer_list
*timer
)
497 struct vtimer_queue
*vq
;
499 /* check if timer is pending */
500 if (!vtimer_pending(timer
))
503 vq
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
504 spin_lock_irqsave(&vq
->lock
, flags
);
506 /* we don't interrupt a running timer, just let it expire! */
507 list_del_init(&timer
->entry
);
509 spin_unlock_irqrestore(&vq
->lock
, flags
);
512 EXPORT_SYMBOL(del_virt_timer
);
515 * Start the virtual CPU timer on the current CPU.
517 void init_cpu_vtimer(void)
519 struct vtimer_queue
*vq
;
521 /* kick the virtual timer */
522 asm volatile ("STCK %0" : "=m" (S390_lowcore
.last_update_clock
));
523 asm volatile ("STPT %0" : "=m" (S390_lowcore
.last_update_timer
));
525 /* initialize per cpu vtimer structure */
526 vq
= &__get_cpu_var(virt_cpu_timer
);
527 INIT_LIST_HEAD(&vq
->list
);
528 spin_lock_init(&vq
->lock
);
530 /* enable cpu timer interrupts */
534 void __init
vtime_init(void)
536 /* request the cpu timer external interrupt */
537 if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt
,
538 &ext_int_info_timer
) != 0)
539 panic("Couldn't request external interrupt 0x1005");
541 /* Enable cpu timer interrupts on the boot cpu. */