blob: fa0726507b3d57152e5af6b11448757050ece7cc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
4 *
5 * S390 version
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
8 */
9
10#include <linux/config.h>
11#include <linux/module.h>
12#include <linux/kernel.h>
13#include <linux/time.h>
14#include <linux/delay.h>
15#include <linux/init.h>
16#include <linux/smp.h>
17#include <linux/types.h>
18#include <linux/timex.h>
19#include <linux/notifier.h>
20#include <linux/kernel_stat.h>
21#include <linux/rcupdate.h>
22#include <linux/posix-timers.h>
23
24#include <asm/s390_ext.h>
25#include <asm/timer.h>
26
27#define VTIMER_MAGIC (TIMER_MAGIC + 1)
28static ext_int_info_t ext_int_info_timer;
29DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
30
31#ifdef CONFIG_VIRT_CPU_ACCOUNTING
32/*
33 * Update process times based on virtual cpu times stored by entry.S
34 * to the lowcore fields user_timer, system_timer & steal_clock.
35 */
36void account_user_vtime(struct task_struct *tsk)
37{
38 cputime_t cputime;
39 __u64 timer, clock;
40 int rcu_user_flag;
41
42 timer = S390_lowcore.last_update_timer;
43 clock = S390_lowcore.last_update_clock;
44 asm volatile (" STPT %0\n" /* Store current cpu timer value */
45 " STCK %1" /* Store current tod clock value */
46 : "=m" (S390_lowcore.last_update_timer),
47 "=m" (S390_lowcore.last_update_clock) );
48 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
49 S390_lowcore.steal_clock += S390_lowcore.last_update_clock - clock;
50
51 cputime = S390_lowcore.user_timer >> 12;
52 rcu_user_flag = cputime != 0;
53 S390_lowcore.user_timer -= cputime << 12;
54 S390_lowcore.steal_clock -= cputime << 12;
55 account_user_time(tsk, cputime);
56
57 cputime = S390_lowcore.system_timer >> 12;
58 S390_lowcore.system_timer -= cputime << 12;
59 S390_lowcore.steal_clock -= cputime << 12;
60 account_system_time(tsk, HARDIRQ_OFFSET, cputime);
61
62 cputime = S390_lowcore.steal_clock;
63 if ((__s64) cputime > 0) {
64 cputime >>= 12;
65 S390_lowcore.steal_clock -= cputime << 12;
66 account_steal_time(tsk, cputime);
67 }
68
69 run_local_timers();
70 if (rcu_pending(smp_processor_id()))
71 rcu_check_callbacks(smp_processor_id(), rcu_user_flag);
72 scheduler_tick();
73 run_posix_cpu_timers(tsk);
74}
75
76/*
77 * Update process times based on virtual cpu times stored by entry.S
78 * to the lowcore fields user_timer, system_timer & steal_clock.
79 */
80void account_system_vtime(struct task_struct *tsk)
81{
82 cputime_t cputime;
83 __u64 timer;
84
85 timer = S390_lowcore.last_update_timer;
86 asm volatile (" STPT %0" /* Store current cpu timer value */
87 : "=m" (S390_lowcore.last_update_timer) );
88 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
89
90 cputime = S390_lowcore.system_timer >> 12;
91 S390_lowcore.system_timer -= cputime << 12;
92 S390_lowcore.steal_clock -= cputime << 12;
93 account_system_time(tsk, 0, cputime);
94}
95
96static inline void set_vtimer(__u64 expires)
97{
98 __u64 timer;
99
100 asm volatile (" STPT %0\n" /* Store current cpu timer value */
101 " SPT %1" /* Set new value immediatly afterwards */
102 : "=m" (timer) : "m" (expires) );
103 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
104 S390_lowcore.last_update_timer = expires;
105
106 /* store expire time for this CPU timer */
107 per_cpu(virt_cpu_timer, smp_processor_id()).to_expire = expires;
108}
109#else
110static inline void set_vtimer(__u64 expires)
111{
112 S390_lowcore.last_update_timer = expires;
113 asm volatile ("SPT %0" : : "m" (S390_lowcore.last_update_timer));
114
115 /* store expire time for this CPU timer */
116 per_cpu(virt_cpu_timer, smp_processor_id()).to_expire = expires;
117}
118#endif
119
120static void start_cpu_timer(void)
121{
122 struct vtimer_queue *vt_list;
123
124 vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
Martin Schwidefsky4b7e0702005-05-01 08:58:57 -0700125
126 /* CPU timer interrupt is pending, don't reprogramm it */
127 if (vt_list->idle & 1LL<<63)
128 return;
129
130 if (!list_empty(&vt_list->list))
131 set_vtimer(vt_list->idle);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132}
133
134static void stop_cpu_timer(void)
135{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 struct vtimer_queue *vt_list;
137
138 vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
139
140 /* nothing to do */
141 if (list_empty(&vt_list->list)) {
142 vt_list->idle = VTIMER_MAX_SLICE;
143 goto fire;
144 }
145
Martin Schwidefsky4b7e0702005-05-01 08:58:57 -0700146 /* store the actual expire value */
147 asm volatile ("STPT %0" : "=m" (vt_list->idle));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148
149 /*
Martin Schwidefsky4b7e0702005-05-01 08:58:57 -0700150 * If the CPU timer is negative we don't reprogramm
151 * it because we will get instantly an interrupt.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 */
Martin Schwidefsky4b7e0702005-05-01 08:58:57 -0700153 if (vt_list->idle & 1LL<<63)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155
Martin Schwidefsky4b7e0702005-05-01 08:58:57 -0700156 vt_list->offset += vt_list->to_expire - vt_list->idle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157
158 /*
159 * We cannot halt the CPU timer, we just write a value that
160 * nearly never expires (only after 71 years) and re-write
161 * the stored expire value if we continue the timer
162 */
163 fire:
164 set_vtimer(VTIMER_MAX_SLICE);
165}
166
167/*
168 * Sorted add to a list. List is linear searched until first bigger
169 * element is found.
170 */
171static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
172{
173 struct vtimer_list *event;
174
175 list_for_each_entry(event, head, entry) {
176 if (event->expires > timer->expires) {
177 list_add_tail(&timer->entry, &event->entry);
178 return;
179 }
180 }
181 list_add_tail(&timer->entry, head);
182}
183
184/*
185 * Do the callback functions of expired vtimer events.
186 * Called from within the interrupt handler.
187 */
188static void do_callbacks(struct list_head *cb_list, struct pt_regs *regs)
189{
190 struct vtimer_queue *vt_list;
191 struct vtimer_list *event, *tmp;
192 void (*fn)(unsigned long, struct pt_regs*);
193 unsigned long data;
194
195 if (list_empty(cb_list))
196 return;
197
198 vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
199
200 list_for_each_entry_safe(event, tmp, cb_list, entry) {
201 fn = event->function;
202 data = event->data;
203 fn(data, regs);
204
205 if (!event->interval)
206 /* delete one shot timer */
207 list_del_init(&event->entry);
208 else {
209 /* move interval timer back to list */
210 spin_lock(&vt_list->lock);
211 list_del_init(&event->entry);
212 list_add_sorted(event, &vt_list->list);
213 spin_unlock(&vt_list->lock);
214 }
215 }
216}
217
218/*
219 * Handler for the virtual CPU timer.
220 */
221static void do_cpu_timer_interrupt(struct pt_regs *regs, __u16 error_code)
222{
223 int cpu;
224 __u64 next, delta;
225 struct vtimer_queue *vt_list;
226 struct vtimer_list *event, *tmp;
227 struct list_head *ptr;
228 /* the callback queue */
229 struct list_head cb_list;
230
231 INIT_LIST_HEAD(&cb_list);
232 cpu = smp_processor_id();
233 vt_list = &per_cpu(virt_cpu_timer, cpu);
234
235 /* walk timer list, fire all expired events */
236 spin_lock(&vt_list->lock);
237
238 if (vt_list->to_expire < VTIMER_MAX_SLICE)
239 vt_list->offset += vt_list->to_expire;
240
241 list_for_each_entry_safe(event, tmp, &vt_list->list, entry) {
242 if (event->expires > vt_list->offset)
243 /* found first unexpired event, leave */
244 break;
245
246 /* re-charge interval timer, we have to add the offset */
247 if (event->interval)
248 event->expires = event->interval + vt_list->offset;
249
250 /* move expired timer to the callback queue */
251 list_move_tail(&event->entry, &cb_list);
252 }
253 spin_unlock(&vt_list->lock);
254 do_callbacks(&cb_list, regs);
255
256 /* next event is first in list */
257 spin_lock(&vt_list->lock);
258 if (!list_empty(&vt_list->list)) {
259 ptr = vt_list->list.next;
260 event = list_entry(ptr, struct vtimer_list, entry);
261 next = event->expires - vt_list->offset;
262
263 /* add the expired time from this interrupt handler
264 * and the callback functions
265 */
266 asm volatile ("STPT %0" : "=m" (delta));
267 delta = 0xffffffffffffffffLL - delta + 1;
268 vt_list->offset += delta;
269 next -= delta;
270 } else {
271 vt_list->offset = 0;
272 next = VTIMER_MAX_SLICE;
273 }
274 spin_unlock(&vt_list->lock);
275 set_vtimer(next);
276}
277
278void init_virt_timer(struct vtimer_list *timer)
279{
280 timer->magic = VTIMER_MAGIC;
281 timer->function = NULL;
282 INIT_LIST_HEAD(&timer->entry);
283 spin_lock_init(&timer->lock);
284}
285EXPORT_SYMBOL(init_virt_timer);
286
287static inline int check_vtimer(struct vtimer_list *timer)
288{
289 if (timer->magic != VTIMER_MAGIC)
290 return -EINVAL;
291 return 0;
292}
293
294static inline int vtimer_pending(struct vtimer_list *timer)
295{
296 return (!list_empty(&timer->entry));
297}
298
299/*
300 * this function should only run on the specified CPU
301 */
302static void internal_add_vtimer(struct vtimer_list *timer)
303{
304 unsigned long flags;
305 __u64 done;
306 struct vtimer_list *event;
307 struct vtimer_queue *vt_list;
308
309 vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
310 spin_lock_irqsave(&vt_list->lock, flags);
311
312 if (timer->cpu != smp_processor_id())
313 printk("internal_add_vtimer: BUG, running on wrong CPU");
314
315 /* if list is empty we only have to set the timer */
316 if (list_empty(&vt_list->list)) {
317 /* reset the offset, this may happen if the last timer was
318 * just deleted by mod_virt_timer and the interrupt
319 * didn't happen until here
320 */
321 vt_list->offset = 0;
322 goto fire;
323 }
324
325 /* save progress */
326 asm volatile ("STPT %0" : "=m" (done));
327
328 /* calculate completed work */
329 done = vt_list->to_expire - done + vt_list->offset;
330 vt_list->offset = 0;
331
332 list_for_each_entry(event, &vt_list->list, entry)
333 event->expires -= done;
334
335 fire:
336 list_add_sorted(timer, &vt_list->list);
337
338 /* get first element, which is the next vtimer slice */
339 event = list_entry(vt_list->list.next, struct vtimer_list, entry);
340
341 set_vtimer(event->expires);
342 spin_unlock_irqrestore(&vt_list->lock, flags);
343 /* release CPU aquired in prepare_vtimer or mod_virt_timer() */
344 put_cpu();
345}
346
347static inline int prepare_vtimer(struct vtimer_list *timer)
348{
349 if (check_vtimer(timer) || !timer->function) {
350 printk("add_virt_timer: uninitialized timer\n");
351 return -EINVAL;
352 }
353
354 if (!timer->expires || timer->expires > VTIMER_MAX_SLICE) {
355 printk("add_virt_timer: invalid timer expire value!\n");
356 return -EINVAL;
357 }
358
359 if (vtimer_pending(timer)) {
360 printk("add_virt_timer: timer pending\n");
361 return -EBUSY;
362 }
363
364 timer->cpu = get_cpu();
365 return 0;
366}
367
368/*
369 * add_virt_timer - add an oneshot virtual CPU timer
370 */
371void add_virt_timer(void *new)
372{
373 struct vtimer_list *timer;
374
375 timer = (struct vtimer_list *)new;
376
377 if (prepare_vtimer(timer) < 0)
378 return;
379
380 timer->interval = 0;
381 internal_add_vtimer(timer);
382}
383EXPORT_SYMBOL(add_virt_timer);
384
385/*
386 * add_virt_timer_int - add an interval virtual CPU timer
387 */
388void add_virt_timer_periodic(void *new)
389{
390 struct vtimer_list *timer;
391
392 timer = (struct vtimer_list *)new;
393
394 if (prepare_vtimer(timer) < 0)
395 return;
396
397 timer->interval = timer->expires;
398 internal_add_vtimer(timer);
399}
400EXPORT_SYMBOL(add_virt_timer_periodic);
401
402/*
403 * If we change a pending timer the function must be called on the CPU
404 * where the timer is running on, e.g. by smp_call_function_on()
405 *
406 * The original mod_timer adds the timer if it is not pending. For compatibility
407 * we do the same. The timer will be added on the current CPU as a oneshot timer.
408 *
409 * returns whether it has modified a pending timer (1) or not (0)
410 */
411int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
412{
413 struct vtimer_queue *vt_list;
414 unsigned long flags;
415 int cpu;
416
417 if (check_vtimer(timer) || !timer->function) {
418 printk("mod_virt_timer: uninitialized timer\n");
419 return -EINVAL;
420 }
421
422 if (!expires || expires > VTIMER_MAX_SLICE) {
423 printk("mod_virt_timer: invalid expire range\n");
424 return -EINVAL;
425 }
426
427 /*
428 * This is a common optimization triggered by the
429 * networking code - if the timer is re-modified
430 * to be the same thing then just return:
431 */
432 if (timer->expires == expires && vtimer_pending(timer))
433 return 1;
434
435 cpu = get_cpu();
436 vt_list = &per_cpu(virt_cpu_timer, cpu);
437
438 /* disable interrupts before test if timer is pending */
439 spin_lock_irqsave(&vt_list->lock, flags);
440
441 /* if timer isn't pending add it on the current CPU */
442 if (!vtimer_pending(timer)) {
443 spin_unlock_irqrestore(&vt_list->lock, flags);
444 /* we do not activate an interval timer with mod_virt_timer */
445 timer->interval = 0;
446 timer->expires = expires;
447 timer->cpu = cpu;
448 internal_add_vtimer(timer);
449 return 0;
450 }
451
452 /* check if we run on the right CPU */
453 if (timer->cpu != cpu) {
454 printk("mod_virt_timer: running on wrong CPU, check your code\n");
455 spin_unlock_irqrestore(&vt_list->lock, flags);
456 put_cpu();
457 return -EINVAL;
458 }
459
460 list_del_init(&timer->entry);
461 timer->expires = expires;
462
463 /* also change the interval if we have an interval timer */
464 if (timer->interval)
465 timer->interval = expires;
466
467 /* the timer can't expire anymore so we can release the lock */
468 spin_unlock_irqrestore(&vt_list->lock, flags);
469 internal_add_vtimer(timer);
470 return 1;
471}
472EXPORT_SYMBOL(mod_virt_timer);
473
474/*
475 * delete a virtual timer
476 *
477 * returns whether the deleted timer was pending (1) or not (0)
478 */
479int del_virt_timer(struct vtimer_list *timer)
480{
481 unsigned long flags;
482 struct vtimer_queue *vt_list;
483
484 if (check_vtimer(timer)) {
485 printk("del_virt_timer: timer not initialized\n");
486 return -EINVAL;
487 }
488
489 /* check if timer is pending */
490 if (!vtimer_pending(timer))
491 return 0;
492
493 vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
494 spin_lock_irqsave(&vt_list->lock, flags);
495
496 /* we don't interrupt a running timer, just let it expire! */
497 list_del_init(&timer->entry);
498
499 /* last timer removed */
500 if (list_empty(&vt_list->list)) {
501 vt_list->to_expire = 0;
502 vt_list->offset = 0;
503 }
504
505 spin_unlock_irqrestore(&vt_list->lock, flags);
506 return 1;
507}
508EXPORT_SYMBOL(del_virt_timer);
509
510/*
511 * Start the virtual CPU timer on the current CPU.
512 */
513void init_cpu_vtimer(void)
514{
515 struct vtimer_queue *vt_list;
516 unsigned long cr0;
517
518 /* kick the virtual timer */
519 S390_lowcore.exit_timer = VTIMER_MAX_SLICE;
520 S390_lowcore.last_update_timer = VTIMER_MAX_SLICE;
521 asm volatile ("SPT %0" : : "m" (S390_lowcore.last_update_timer));
522 asm volatile ("STCK %0" : "=m" (S390_lowcore.last_update_clock));
523 __ctl_store(cr0, 0, 0);
524 cr0 |= 0x400;
525 __ctl_load(cr0, 0, 0);
526
527 vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
528 INIT_LIST_HEAD(&vt_list->list);
529 spin_lock_init(&vt_list->lock);
530 vt_list->to_expire = 0;
531 vt_list->offset = 0;
532 vt_list->idle = 0;
533
534}
535
536static int vtimer_idle_notify(struct notifier_block *self,
537 unsigned long action, void *hcpu)
538{
539 switch (action) {
540 case CPU_IDLE:
541 stop_cpu_timer();
542 break;
543 case CPU_NOT_IDLE:
544 start_cpu_timer();
545 break;
546 }
547 return NOTIFY_OK;
548}
549
550static struct notifier_block vtimer_idle_nb = {
551 .notifier_call = vtimer_idle_notify,
552};
553
554void __init vtime_init(void)
555{
556 /* request the cpu timer external interrupt */
557 if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt,
558 &ext_int_info_timer) != 0)
559 panic("Couldn't request external interrupt 0x1005");
560
561 if (register_idle_notifier(&vtimer_idle_nb))
562 panic("Couldn't register idle notifier");
563
564 init_cpu_vtimer();
565}
566