2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
10 #include <linux/config.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/time.h>
14 #include <linux/delay.h>
15 #include <linux/init.h>
16 #include <linux/smp.h>
17 #include <linux/types.h>
18 #include <linux/timex.h>
19 #include <linux/notifier.h>
20 #include <linux/kernel_stat.h>
21 #include <linux/rcupdate.h>
22 #include <linux/posix-timers.h>
24 #include <asm/s390_ext.h>
25 #include <asm/timer.h>
27 static ext_int_info_t ext_int_info_timer
;
28 DEFINE_PER_CPU(struct vtimer_queue
, virt_cpu_timer
);
30 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
32 * Update process times based on virtual cpu times stored by entry.S
33 * to the lowcore fields user_timer, system_timer & steal_clock.
35 void account_user_vtime(struct task_struct
*tsk
)
41 timer
= S390_lowcore
.last_update_timer
;
42 clock
= S390_lowcore
.last_update_clock
;
43 asm volatile (" STPT %0\n" /* Store current cpu timer value */
44 " STCK %1" /* Store current tod clock value */
45 : "=m" (S390_lowcore
.last_update_timer
),
46 "=m" (S390_lowcore
.last_update_clock
) );
47 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
48 S390_lowcore
.steal_clock
+= S390_lowcore
.last_update_clock
- clock
;
50 cputime
= S390_lowcore
.user_timer
>> 12;
51 rcu_user_flag
= cputime
!= 0;
52 S390_lowcore
.user_timer
-= cputime
<< 12;
53 S390_lowcore
.steal_clock
-= cputime
<< 12;
54 account_user_time(tsk
, cputime
);
56 cputime
= S390_lowcore
.system_timer
>> 12;
57 S390_lowcore
.system_timer
-= cputime
<< 12;
58 S390_lowcore
.steal_clock
-= cputime
<< 12;
59 account_system_time(tsk
, HARDIRQ_OFFSET
, cputime
);
61 cputime
= S390_lowcore
.steal_clock
;
62 if ((__s64
) cputime
> 0) {
64 S390_lowcore
.steal_clock
-= cputime
<< 12;
65 account_steal_time(tsk
, cputime
);
69 if (rcu_pending(smp_processor_id()))
70 rcu_check_callbacks(smp_processor_id(), rcu_user_flag
);
72 run_posix_cpu_timers(tsk
);
76 * Update process times based on virtual cpu times stored by entry.S
77 * to the lowcore fields user_timer, system_timer & steal_clock.
79 void account_system_vtime(struct task_struct
*tsk
)
84 timer
= S390_lowcore
.last_update_timer
;
85 asm volatile (" STPT %0" /* Store current cpu timer value */
86 : "=m" (S390_lowcore
.last_update_timer
) );
87 S390_lowcore
.system_timer
+= timer
- S390_lowcore
.last_update_timer
;
89 cputime
= S390_lowcore
.system_timer
>> 12;
90 S390_lowcore
.system_timer
-= cputime
<< 12;
91 S390_lowcore
.steal_clock
-= cputime
<< 12;
92 account_system_time(tsk
, 0, cputime
);
95 static inline void set_vtimer(__u64 expires
)
99 asm volatile (" STPT %0\n" /* Store current cpu timer value */
100 " SPT %1" /* Set new value immediatly afterwards */
101 : "=m" (timer
) : "m" (expires
) );
102 S390_lowcore
.system_timer
+= S390_lowcore
.last_update_timer
- timer
;
103 S390_lowcore
.last_update_timer
= expires
;
105 /* store expire time for this CPU timer */
106 per_cpu(virt_cpu_timer
, smp_processor_id()).to_expire
= expires
;
109 static inline void set_vtimer(__u64 expires
)
111 S390_lowcore
.last_update_timer
= expires
;
112 asm volatile ("SPT %0" : : "m" (S390_lowcore
.last_update_timer
));
114 /* store expire time for this CPU timer */
115 per_cpu(virt_cpu_timer
, smp_processor_id()).to_expire
= expires
;
119 static void start_cpu_timer(void)
121 struct vtimer_queue
*vt_list
;
123 vt_list
= &per_cpu(virt_cpu_timer
, smp_processor_id());
125 /* CPU timer interrupt is pending, don't reprogramm it */
126 if (vt_list
->idle
& 1LL<<63)
129 if (!list_empty(&vt_list
->list
))
130 set_vtimer(vt_list
->idle
);
133 static void stop_cpu_timer(void)
135 struct vtimer_queue
*vt_list
;
137 vt_list
= &per_cpu(virt_cpu_timer
, smp_processor_id());
140 if (list_empty(&vt_list
->list
)) {
141 vt_list
->idle
= VTIMER_MAX_SLICE
;
145 /* store the actual expire value */
146 asm volatile ("STPT %0" : "=m" (vt_list
->idle
));
149 * If the CPU timer is negative we don't reprogramm
150 * it because we will get instantly an interrupt.
152 if (vt_list
->idle
& 1LL<<63)
155 vt_list
->offset
+= vt_list
->to_expire
- vt_list
->idle
;
158 * We cannot halt the CPU timer, we just write a value that
159 * nearly never expires (only after 71 years) and re-write
160 * the stored expire value if we continue the timer
163 set_vtimer(VTIMER_MAX_SLICE
);
167 * Sorted add to a list. List is linear searched until first bigger
170 static void list_add_sorted(struct vtimer_list
*timer
, struct list_head
*head
)
172 struct vtimer_list
*event
;
174 list_for_each_entry(event
, head
, entry
) {
175 if (event
->expires
> timer
->expires
) {
176 list_add_tail(&timer
->entry
, &event
->entry
);
180 list_add_tail(&timer
->entry
, head
);
184 * Do the callback functions of expired vtimer events.
185 * Called from within the interrupt handler.
187 static void do_callbacks(struct list_head
*cb_list
, struct pt_regs
*regs
)
189 struct vtimer_queue
*vt_list
;
190 struct vtimer_list
*event
, *tmp
;
191 void (*fn
)(unsigned long, struct pt_regs
*);
194 if (list_empty(cb_list
))
197 vt_list
= &per_cpu(virt_cpu_timer
, smp_processor_id());
199 list_for_each_entry_safe(event
, tmp
, cb_list
, entry
) {
200 fn
= event
->function
;
204 if (!event
->interval
)
205 /* delete one shot timer */
206 list_del_init(&event
->entry
);
208 /* move interval timer back to list */
209 spin_lock(&vt_list
->lock
);
210 list_del_init(&event
->entry
);
211 list_add_sorted(event
, &vt_list
->list
);
212 spin_unlock(&vt_list
->lock
);
218 * Handler for the virtual CPU timer.
220 static void do_cpu_timer_interrupt(struct pt_regs
*regs
, __u16 error_code
)
224 struct vtimer_queue
*vt_list
;
225 struct vtimer_list
*event
, *tmp
;
226 struct list_head
*ptr
;
227 /* the callback queue */
228 struct list_head cb_list
;
230 INIT_LIST_HEAD(&cb_list
);
231 cpu
= smp_processor_id();
232 vt_list
= &per_cpu(virt_cpu_timer
, cpu
);
234 /* walk timer list, fire all expired events */
235 spin_lock(&vt_list
->lock
);
237 if (vt_list
->to_expire
< VTIMER_MAX_SLICE
)
238 vt_list
->offset
+= vt_list
->to_expire
;
240 list_for_each_entry_safe(event
, tmp
, &vt_list
->list
, entry
) {
241 if (event
->expires
> vt_list
->offset
)
242 /* found first unexpired event, leave */
245 /* re-charge interval timer, we have to add the offset */
247 event
->expires
= event
->interval
+ vt_list
->offset
;
249 /* move expired timer to the callback queue */
250 list_move_tail(&event
->entry
, &cb_list
);
252 spin_unlock(&vt_list
->lock
);
253 do_callbacks(&cb_list
, regs
);
255 /* next event is first in list */
256 spin_lock(&vt_list
->lock
);
257 if (!list_empty(&vt_list
->list
)) {
258 ptr
= vt_list
->list
.next
;
259 event
= list_entry(ptr
, struct vtimer_list
, entry
);
260 next
= event
->expires
- vt_list
->offset
;
262 /* add the expired time from this interrupt handler
263 * and the callback functions
265 asm volatile ("STPT %0" : "=m" (delta
));
266 delta
= 0xffffffffffffffffLL
- delta
+ 1;
267 vt_list
->offset
+= delta
;
271 next
= VTIMER_MAX_SLICE
;
273 spin_unlock(&vt_list
->lock
);
277 void init_virt_timer(struct vtimer_list
*timer
)
279 timer
->function
= NULL
;
280 INIT_LIST_HEAD(&timer
->entry
);
281 spin_lock_init(&timer
->lock
);
283 EXPORT_SYMBOL(init_virt_timer
);
285 static inline int vtimer_pending(struct vtimer_list
*timer
)
287 return (!list_empty(&timer
->entry
));
291 * this function should only run on the specified CPU
293 static void internal_add_vtimer(struct vtimer_list
*timer
)
297 struct vtimer_list
*event
;
298 struct vtimer_queue
*vt_list
;
300 vt_list
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
301 spin_lock_irqsave(&vt_list
->lock
, flags
);
303 if (timer
->cpu
!= smp_processor_id())
304 printk("internal_add_vtimer: BUG, running on wrong CPU");
306 /* if list is empty we only have to set the timer */
307 if (list_empty(&vt_list
->list
)) {
308 /* reset the offset, this may happen if the last timer was
309 * just deleted by mod_virt_timer and the interrupt
310 * didn't happen until here
317 asm volatile ("STPT %0" : "=m" (done
));
319 /* calculate completed work */
320 done
= vt_list
->to_expire
- done
+ vt_list
->offset
;
323 list_for_each_entry(event
, &vt_list
->list
, entry
)
324 event
->expires
-= done
;
327 list_add_sorted(timer
, &vt_list
->list
);
329 /* get first element, which is the next vtimer slice */
330 event
= list_entry(vt_list
->list
.next
, struct vtimer_list
, entry
);
332 set_vtimer(event
->expires
);
333 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
334 /* release CPU aquired in prepare_vtimer or mod_virt_timer() */
338 static inline int prepare_vtimer(struct vtimer_list
*timer
)
340 if (!timer
->function
) {
341 printk("add_virt_timer: uninitialized timer\n");
345 if (!timer
->expires
|| timer
->expires
> VTIMER_MAX_SLICE
) {
346 printk("add_virt_timer: invalid timer expire value!\n");
350 if (vtimer_pending(timer
)) {
351 printk("add_virt_timer: timer pending\n");
355 timer
->cpu
= get_cpu();
360 * add_virt_timer - add an oneshot virtual CPU timer
362 void add_virt_timer(void *new)
364 struct vtimer_list
*timer
;
366 timer
= (struct vtimer_list
*)new;
368 if (prepare_vtimer(timer
) < 0)
372 internal_add_vtimer(timer
);
374 EXPORT_SYMBOL(add_virt_timer
);
377 * add_virt_timer_int - add an interval virtual CPU timer
379 void add_virt_timer_periodic(void *new)
381 struct vtimer_list
*timer
;
383 timer
= (struct vtimer_list
*)new;
385 if (prepare_vtimer(timer
) < 0)
388 timer
->interval
= timer
->expires
;
389 internal_add_vtimer(timer
);
391 EXPORT_SYMBOL(add_virt_timer_periodic
);
394 * If we change a pending timer the function must be called on the CPU
395 * where the timer is running on, e.g. by smp_call_function_on()
397 * The original mod_timer adds the timer if it is not pending. For compatibility
398 * we do the same. The timer will be added on the current CPU as a oneshot timer.
400 * returns whether it has modified a pending timer (1) or not (0)
402 int mod_virt_timer(struct vtimer_list
*timer
, __u64 expires
)
404 struct vtimer_queue
*vt_list
;
408 if (!timer
->function
) {
409 printk("mod_virt_timer: uninitialized timer\n");
413 if (!expires
|| expires
> VTIMER_MAX_SLICE
) {
414 printk("mod_virt_timer: invalid expire range\n");
419 * This is a common optimization triggered by the
420 * networking code - if the timer is re-modified
421 * to be the same thing then just return:
423 if (timer
->expires
== expires
&& vtimer_pending(timer
))
427 vt_list
= &per_cpu(virt_cpu_timer
, cpu
);
429 /* disable interrupts before test if timer is pending */
430 spin_lock_irqsave(&vt_list
->lock
, flags
);
432 /* if timer isn't pending add it on the current CPU */
433 if (!vtimer_pending(timer
)) {
434 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
435 /* we do not activate an interval timer with mod_virt_timer */
437 timer
->expires
= expires
;
439 internal_add_vtimer(timer
);
443 /* check if we run on the right CPU */
444 if (timer
->cpu
!= cpu
) {
445 printk("mod_virt_timer: running on wrong CPU, check your code\n");
446 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
451 list_del_init(&timer
->entry
);
452 timer
->expires
= expires
;
454 /* also change the interval if we have an interval timer */
456 timer
->interval
= expires
;
458 /* the timer can't expire anymore so we can release the lock */
459 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
460 internal_add_vtimer(timer
);
463 EXPORT_SYMBOL(mod_virt_timer
);
466 * delete a virtual timer
468 * returns whether the deleted timer was pending (1) or not (0)
470 int del_virt_timer(struct vtimer_list
*timer
)
473 struct vtimer_queue
*vt_list
;
475 /* check if timer is pending */
476 if (!vtimer_pending(timer
))
479 vt_list
= &per_cpu(virt_cpu_timer
, timer
->cpu
);
480 spin_lock_irqsave(&vt_list
->lock
, flags
);
482 /* we don't interrupt a running timer, just let it expire! */
483 list_del_init(&timer
->entry
);
485 /* last timer removed */
486 if (list_empty(&vt_list
->list
)) {
487 vt_list
->to_expire
= 0;
491 spin_unlock_irqrestore(&vt_list
->lock
, flags
);
494 EXPORT_SYMBOL(del_virt_timer
);
497 * Start the virtual CPU timer on the current CPU.
499 void init_cpu_vtimer(void)
501 struct vtimer_queue
*vt_list
;
504 /* kick the virtual timer */
505 S390_lowcore
.exit_timer
= VTIMER_MAX_SLICE
;
506 S390_lowcore
.last_update_timer
= VTIMER_MAX_SLICE
;
507 asm volatile ("SPT %0" : : "m" (S390_lowcore
.last_update_timer
));
508 asm volatile ("STCK %0" : "=m" (S390_lowcore
.last_update_clock
));
509 __ctl_store(cr0
, 0, 0);
511 __ctl_load(cr0
, 0, 0);
513 vt_list
= &per_cpu(virt_cpu_timer
, smp_processor_id());
514 INIT_LIST_HEAD(&vt_list
->list
);
515 spin_lock_init(&vt_list
->lock
);
516 vt_list
->to_expire
= 0;
522 static int vtimer_idle_notify(struct notifier_block
*self
,
523 unsigned long action
, void *hcpu
)
536 static struct notifier_block vtimer_idle_nb
= {
537 .notifier_call
= vtimer_idle_notify
,
540 void __init
vtime_init(void)
542 /* request the cpu timer external interrupt */
543 if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt
,
544 &ext_int_info_timer
) != 0)
545 panic("Couldn't request external interrupt 0x1005");
547 if (register_idle_notifier(&vtimer_idle_nb
))
548 panic("Couldn't register idle notifier");