blob: 5db5a8d2681168cfcf9c571d2da69629245c92ba [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/timer.c
3 *
john stultz85240702007-05-08 00:27:59 -07004 * Kernel internal timers, basic process system calls
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 *
6 * Copyright (C) 1991, 1992 Linus Torvalds
7 *
8 * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
9 *
10 * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
11 * "A Kernel Model for Precision Timekeeping" by Dave Mills
12 * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
13 * serialize accesses to xtime/lost_ticks).
14 * Copyright (C) 1998 Andrea Arcangeli
15 * 1999-03-10 Improved NTP compatibility by Ulrich Windl
16 * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
17 * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
18 * Copyright (C) 2000, 2001, 2002 Ingo Molnar
19 * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
20 */
21
22#include <linux/kernel_stat.h>
23#include <linux/module.h>
24#include <linux/interrupt.h>
25#include <linux/percpu.h>
26#include <linux/init.h>
27#include <linux/mm.h>
28#include <linux/swap.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070029#include <linux/pid_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/notifier.h>
31#include <linux/thread_info.h>
32#include <linux/time.h>
33#include <linux/jiffies.h>
34#include <linux/posix-timers.h>
35#include <linux/cpu.h>
36#include <linux/syscalls.h>
Adrian Bunk97a41e22006-01-08 01:02:17 -080037#include <linux/delay.h>
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080038#include <linux/tick.h>
Ingo Molnar82f67cd2007-02-16 01:28:13 -080039#include <linux/kallsyms.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020040#include <linux/perf_event.h>
Arun R Bharadwajeea08f32009-04-16 12:16:41 +053041#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
43#include <asm/uaccess.h>
44#include <asm/unistd.h>
45#include <asm/div64.h>
46#include <asm/timex.h>
47#include <asm/io.h>
48
Xiao Guangrong2b022e32009-08-10 10:48:59 +080049#define CREATE_TRACE_POINTS
50#include <trace/events/timer.h>
51
Thomas Gleixnerecea8d12005-10-30 15:03:00 -080052u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
53
54EXPORT_SYMBOL(jiffies_64);
55
Linus Torvalds1da177e2005-04-16 15:20:36 -070056/*
57 * per-CPU timer vector definitions:
58 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070059#define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6)
60#define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8)
61#define TVN_SIZE (1 << TVN_BITS)
62#define TVR_SIZE (1 << TVR_BITS)
63#define TVN_MASK (TVN_SIZE - 1)
64#define TVR_MASK (TVR_SIZE - 1)
65
Pavel Macheka6fa8e52008-01-30 13:30:00 +010066struct tvec {
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 struct list_head vec[TVN_SIZE];
Pavel Macheka6fa8e52008-01-30 13:30:00 +010068};
Linus Torvalds1da177e2005-04-16 15:20:36 -070069
Pavel Macheka6fa8e52008-01-30 13:30:00 +010070struct tvec_root {
Linus Torvalds1da177e2005-04-16 15:20:36 -070071 struct list_head vec[TVR_SIZE];
Pavel Macheka6fa8e52008-01-30 13:30:00 +010072};
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Pavel Macheka6fa8e52008-01-30 13:30:00 +010074struct tvec_base {
Oleg Nesterov3691c512006-03-31 02:30:30 -080075 spinlock_t lock;
76 struct timer_list *running_timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 unsigned long timer_jiffies;
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +020078 unsigned long next_timer;
Pavel Macheka6fa8e52008-01-30 13:30:00 +010079 struct tvec_root tv1;
80 struct tvec tv2;
81 struct tvec tv3;
82 struct tvec tv4;
83 struct tvec tv5;
Venki Pallipadi6e453a62007-05-08 00:27:44 -070084} ____cacheline_aligned;
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Pavel Macheka6fa8e52008-01-30 13:30:00 +010086struct tvec_base boot_tvec_bases;
Oleg Nesterov3691c512006-03-31 02:30:30 -080087EXPORT_SYMBOL(boot_tvec_bases);
Pavel Macheka6fa8e52008-01-30 13:30:00 +010088static DEFINE_PER_CPU(struct tvec_base *, tvec_bases) = &boot_tvec_bases;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089
Venki Pallipadi6e453a62007-05-08 00:27:44 -070090/*
Pavel Macheka6fa8e52008-01-30 13:30:00 +010091 * Note that all tvec_bases are 2 byte aligned and lower bit of
Venki Pallipadi6e453a62007-05-08 00:27:44 -070092 * base in timer_list is guaranteed to be zero. Use the LSB for
93 * the new flag to indicate whether the timer is deferrable
94 */
95#define TBASE_DEFERRABLE_FLAG (0x1)
96
97/* Functions below help us manage 'deferrable' flag */
Pavel Macheka6fa8e52008-01-30 13:30:00 +010098static inline unsigned int tbase_get_deferrable(struct tvec_base *base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -070099{
akpm@linux-foundation.orge9910842007-05-10 03:16:01 -0700100 return ((unsigned int)(unsigned long)base & TBASE_DEFERRABLE_FLAG);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700101}
102
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100103static inline struct tvec_base *tbase_get_base(struct tvec_base *base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700104{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100105 return ((struct tvec_base *)((unsigned long)base & ~TBASE_DEFERRABLE_FLAG));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700106}
107
108static inline void timer_set_deferrable(struct timer_list *timer)
109{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100110 timer->base = ((struct tvec_base *)((unsigned long)(timer->base) |
Thomas Gleixner68194572007-07-19 01:49:16 -0700111 TBASE_DEFERRABLE_FLAG));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700112}
113
114static inline void
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100115timer_set_base(struct timer_list *timer, struct tvec_base *new_base)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700116{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100117 timer->base = (struct tvec_base *)((unsigned long)(new_base) |
Thomas Gleixner68194572007-07-19 01:49:16 -0700118 tbase_get_deferrable(timer->base));
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700119}
120
Alan Stern9c133c42008-11-06 08:42:48 +0100121static unsigned long round_jiffies_common(unsigned long j, int cpu,
122 bool force_up)
123{
124 int rem;
125 unsigned long original = j;
126
127 /*
128 * We don't want all cpus firing their timers at once hitting the
129 * same lock or cachelines, so we skew each extra cpu with an extra
130 * 3 jiffies. This 3 jiffies came originally from the mm/ code which
131 * already did this.
132 * The skew is done by adding 3*cpunr, then round, then subtract this
133 * extra offset again.
134 */
135 j += cpu * 3;
136
137 rem = j % HZ;
138
139 /*
140 * If the target jiffie is just after a whole second (which can happen
141 * due to delays of the timer irq, long irq off times etc etc) then
142 * we should round down to the whole second, not up. Use 1/4th second
143 * as cutoff for this rounding as an extreme upper bound for this.
144 * But never round down if @force_up is set.
145 */
146 if (rem < HZ/4 && !force_up) /* round down */
147 j = j - rem;
148 else /* round up */
149 j = j - rem + HZ;
150
151 /* now that we have rounded, subtract the extra skew again */
152 j -= cpu * 3;
153
154 if (j <= jiffies) /* rounding ate our timeout entirely; */
155 return original;
156 return j;
157}
158
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800159/**
160 * __round_jiffies - function to round jiffies to a full second
161 * @j: the time in (absolute) jiffies that should be rounded
162 * @cpu: the processor number on which the timeout will happen
163 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800164 * __round_jiffies() rounds an absolute time in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800165 * up or down to (approximately) full seconds. This is useful for timers
166 * for which the exact time they fire does not matter too much, as long as
167 * they fire approximately every X seconds.
168 *
169 * By rounding these timers to whole seconds, all such timers will fire
170 * at the same time, rather than at various times spread out. The goal
171 * of this is to have the CPU wake up less, which saves power.
172 *
173 * The exact rounding is skewed for each processor to avoid all
174 * processors firing at the exact same time, which could lead
175 * to lock contention or spurious cache line bouncing.
176 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800177 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800178 */
179unsigned long __round_jiffies(unsigned long j, int cpu)
180{
Alan Stern9c133c42008-11-06 08:42:48 +0100181 return round_jiffies_common(j, cpu, false);
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800182}
183EXPORT_SYMBOL_GPL(__round_jiffies);
184
185/**
186 * __round_jiffies_relative - function to round jiffies to a full second
187 * @j: the time in (relative) jiffies that should be rounded
188 * @cpu: the processor number on which the timeout will happen
189 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800190 * __round_jiffies_relative() rounds a time delta in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800191 * up or down to (approximately) full seconds. This is useful for timers
192 * for which the exact time they fire does not matter too much, as long as
193 * they fire approximately every X seconds.
194 *
195 * By rounding these timers to whole seconds, all such timers will fire
196 * at the same time, rather than at various times spread out. The goal
197 * of this is to have the CPU wake up less, which saves power.
198 *
199 * The exact rounding is skewed for each processor to avoid all
200 * processors firing at the exact same time, which could lead
201 * to lock contention or spurious cache line bouncing.
202 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800203 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800204 */
205unsigned long __round_jiffies_relative(unsigned long j, int cpu)
206{
Alan Stern9c133c42008-11-06 08:42:48 +0100207 unsigned long j0 = jiffies;
208
209 /* Use j0 because jiffies might change while we run */
210 return round_jiffies_common(j + j0, cpu, false) - j0;
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800211}
212EXPORT_SYMBOL_GPL(__round_jiffies_relative);
213
214/**
215 * round_jiffies - function to round jiffies to a full second
216 * @j: the time in (absolute) jiffies that should be rounded
217 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800218 * round_jiffies() rounds an absolute time in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800219 * up or down to (approximately) full seconds. This is useful for timers
220 * for which the exact time they fire does not matter too much, as long as
221 * they fire approximately every X seconds.
222 *
223 * By rounding these timers to whole seconds, all such timers will fire
224 * at the same time, rather than at various times spread out. The goal
225 * of this is to have the CPU wake up less, which saves power.
226 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800227 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800228 */
229unsigned long round_jiffies(unsigned long j)
230{
Alan Stern9c133c42008-11-06 08:42:48 +0100231 return round_jiffies_common(j, raw_smp_processor_id(), false);
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800232}
233EXPORT_SYMBOL_GPL(round_jiffies);
234
235/**
236 * round_jiffies_relative - function to round jiffies to a full second
237 * @j: the time in (relative) jiffies that should be rounded
238 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800239 * round_jiffies_relative() rounds a time delta in the future (in jiffies)
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800240 * up or down to (approximately) full seconds. This is useful for timers
241 * for which the exact time they fire does not matter too much, as long as
242 * they fire approximately every X seconds.
243 *
244 * By rounding these timers to whole seconds, all such timers will fire
245 * at the same time, rather than at various times spread out. The goal
246 * of this is to have the CPU wake up less, which saves power.
247 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800248 * The return value is the rounded version of the @j parameter.
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800249 */
250unsigned long round_jiffies_relative(unsigned long j)
251{
252 return __round_jiffies_relative(j, raw_smp_processor_id());
253}
254EXPORT_SYMBOL_GPL(round_jiffies_relative);
255
Alan Stern9c133c42008-11-06 08:42:48 +0100256/**
257 * __round_jiffies_up - function to round jiffies up to a full second
258 * @j: the time in (absolute) jiffies that should be rounded
259 * @cpu: the processor number on which the timeout will happen
260 *
261 * This is the same as __round_jiffies() except that it will never
262 * round down. This is useful for timeouts for which the exact time
263 * of firing does not matter too much, as long as they don't fire too
264 * early.
265 */
266unsigned long __round_jiffies_up(unsigned long j, int cpu)
267{
268 return round_jiffies_common(j, cpu, true);
269}
270EXPORT_SYMBOL_GPL(__round_jiffies_up);
271
272/**
273 * __round_jiffies_up_relative - function to round jiffies up to a full second
274 * @j: the time in (relative) jiffies that should be rounded
275 * @cpu: the processor number on which the timeout will happen
276 *
277 * This is the same as __round_jiffies_relative() except that it will never
278 * round down. This is useful for timeouts for which the exact time
279 * of firing does not matter too much, as long as they don't fire too
280 * early.
281 */
282unsigned long __round_jiffies_up_relative(unsigned long j, int cpu)
283{
284 unsigned long j0 = jiffies;
285
286 /* Use j0 because jiffies might change while we run */
287 return round_jiffies_common(j + j0, cpu, true) - j0;
288}
289EXPORT_SYMBOL_GPL(__round_jiffies_up_relative);
290
291/**
292 * round_jiffies_up - function to round jiffies up to a full second
293 * @j: the time in (absolute) jiffies that should be rounded
294 *
295 * This is the same as round_jiffies() except that it will never
296 * round down. This is useful for timeouts for which the exact time
297 * of firing does not matter too much, as long as they don't fire too
298 * early.
299 */
300unsigned long round_jiffies_up(unsigned long j)
301{
302 return round_jiffies_common(j, raw_smp_processor_id(), true);
303}
304EXPORT_SYMBOL_GPL(round_jiffies_up);
305
306/**
307 * round_jiffies_up_relative - function to round jiffies up to a full second
308 * @j: the time in (relative) jiffies that should be rounded
309 *
310 * This is the same as round_jiffies_relative() except that it will never
311 * round down. This is useful for timeouts for which the exact time
312 * of firing does not matter too much, as long as they don't fire too
313 * early.
314 */
315unsigned long round_jiffies_up_relative(unsigned long j)
316{
317 return __round_jiffies_up_relative(j, raw_smp_processor_id());
318}
319EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
320
Arjan van de Ven4c36a5d2006-12-10 02:21:24 -0800321
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100322static inline void set_running_timer(struct tvec_base *base,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 struct timer_list *timer)
324{
325#ifdef CONFIG_SMP
Oleg Nesterov3691c512006-03-31 02:30:30 -0800326 base->running_timer = timer;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327#endif
328}
329
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100330static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331{
332 unsigned long expires = timer->expires;
333 unsigned long idx = expires - base->timer_jiffies;
334 struct list_head *vec;
335
336 if (idx < TVR_SIZE) {
337 int i = expires & TVR_MASK;
338 vec = base->tv1.vec + i;
339 } else if (idx < 1 << (TVR_BITS + TVN_BITS)) {
340 int i = (expires >> TVR_BITS) & TVN_MASK;
341 vec = base->tv2.vec + i;
342 } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) {
343 int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK;
344 vec = base->tv3.vec + i;
345 } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) {
346 int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK;
347 vec = base->tv4.vec + i;
348 } else if ((signed long) idx < 0) {
349 /*
350 * Can happen if you add a timer with expires == jiffies,
351 * or you set a timer to go off in the past
352 */
353 vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK);
354 } else {
355 int i;
356 /* If the timeout is larger than 0xffffffff on 64-bit
357 * architectures then we use the maximum timeout:
358 */
359 if (idx > 0xffffffffUL) {
360 idx = 0xffffffffUL;
361 expires = idx + base->timer_jiffies;
362 }
363 i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK;
364 vec = base->tv5.vec + i;
365 }
366 /*
367 * Timers are FIFO:
368 */
369 list_add_tail(&timer->entry, vec);
370}
371
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800372#ifdef CONFIG_TIMER_STATS
373void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr)
374{
375 if (timer->start_site)
376 return;
377
378 timer->start_site = addr;
379 memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
380 timer->start_pid = current->pid;
381}
Venki Pallipadic5c061b82007-07-15 23:40:30 -0700382
383static void timer_stats_account_timer(struct timer_list *timer)
384{
385 unsigned int flag = 0;
386
Heiko Carstens507e1232009-06-23 17:38:15 +0200387 if (likely(!timer->start_site))
388 return;
Venki Pallipadic5c061b82007-07-15 23:40:30 -0700389 if (unlikely(tbase_get_deferrable(timer->base)))
390 flag |= TIMER_STATS_FLAG_DEFERRABLE;
391
392 timer_stats_update_stats(timer, timer->start_pid, timer->start_site,
393 timer->function, timer->start_comm, flag);
394}
395
396#else
397static void timer_stats_account_timer(struct timer_list *timer) {}
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800398#endif
399
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700400#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
401
402static struct debug_obj_descr timer_debug_descr;
403
404/*
405 * fixup_init is called when:
406 * - an active object is initialized
407 */
408static int timer_fixup_init(void *addr, enum debug_obj_state state)
409{
410 struct timer_list *timer = addr;
411
412 switch (state) {
413 case ODEBUG_STATE_ACTIVE:
414 del_timer_sync(timer);
415 debug_object_init(timer, &timer_debug_descr);
416 return 1;
417 default:
418 return 0;
419 }
420}
421
422/*
423 * fixup_activate is called when:
424 * - an active object is activated
425 * - an unknown object is activated (might be a statically initialized object)
426 */
427static int timer_fixup_activate(void *addr, enum debug_obj_state state)
428{
429 struct timer_list *timer = addr;
430
431 switch (state) {
432
433 case ODEBUG_STATE_NOTAVAILABLE:
434 /*
435 * This is not really a fixup. The timer was
436 * statically initialized. We just make sure that it
437 * is tracked in the object tracker.
438 */
439 if (timer->entry.next == NULL &&
440 timer->entry.prev == TIMER_ENTRY_STATIC) {
441 debug_object_init(timer, &timer_debug_descr);
442 debug_object_activate(timer, &timer_debug_descr);
443 return 0;
444 } else {
445 WARN_ON_ONCE(1);
446 }
447 return 0;
448
449 case ODEBUG_STATE_ACTIVE:
450 WARN_ON(1);
451
452 default:
453 return 0;
454 }
455}
456
457/*
458 * fixup_free is called when:
459 * - an active object is freed
460 */
461static int timer_fixup_free(void *addr, enum debug_obj_state state)
462{
463 struct timer_list *timer = addr;
464
465 switch (state) {
466 case ODEBUG_STATE_ACTIVE:
467 del_timer_sync(timer);
468 debug_object_free(timer, &timer_debug_descr);
469 return 1;
470 default:
471 return 0;
472 }
473}
474
475static struct debug_obj_descr timer_debug_descr = {
476 .name = "timer_list",
477 .fixup_init = timer_fixup_init,
478 .fixup_activate = timer_fixup_activate,
479 .fixup_free = timer_fixup_free,
480};
481
482static inline void debug_timer_init(struct timer_list *timer)
483{
484 debug_object_init(timer, &timer_debug_descr);
485}
486
487static inline void debug_timer_activate(struct timer_list *timer)
488{
489 debug_object_activate(timer, &timer_debug_descr);
490}
491
492static inline void debug_timer_deactivate(struct timer_list *timer)
493{
494 debug_object_deactivate(timer, &timer_debug_descr);
495}
496
497static inline void debug_timer_free(struct timer_list *timer)
498{
499 debug_object_free(timer, &timer_debug_descr);
500}
501
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100502static void __init_timer(struct timer_list *timer,
503 const char *name,
504 struct lock_class_key *key);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700505
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100506void init_timer_on_stack_key(struct timer_list *timer,
507 const char *name,
508 struct lock_class_key *key)
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700509{
510 debug_object_init_on_stack(timer, &timer_debug_descr);
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100511 __init_timer(timer, name, key);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700512}
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100513EXPORT_SYMBOL_GPL(init_timer_on_stack_key);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700514
515void destroy_timer_on_stack(struct timer_list *timer)
516{
517 debug_object_free(timer, &timer_debug_descr);
518}
519EXPORT_SYMBOL_GPL(destroy_timer_on_stack);
520
521#else
522static inline void debug_timer_init(struct timer_list *timer) { }
523static inline void debug_timer_activate(struct timer_list *timer) { }
524static inline void debug_timer_deactivate(struct timer_list *timer) { }
525#endif
526
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800527static inline void debug_init(struct timer_list *timer)
528{
529 debug_timer_init(timer);
530 trace_timer_init(timer);
531}
532
533static inline void
534debug_activate(struct timer_list *timer, unsigned long expires)
535{
536 debug_timer_activate(timer);
537 trace_timer_start(timer, expires);
538}
539
540static inline void debug_deactivate(struct timer_list *timer)
541{
542 debug_timer_deactivate(timer);
543 trace_timer_cancel(timer);
544}
545
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100546static void __init_timer(struct timer_list *timer,
547 const char *name,
548 struct lock_class_key *key)
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700549{
550 timer->entry.next = NULL;
551 timer->base = __raw_get_cpu_var(tvec_bases);
552#ifdef CONFIG_TIMER_STATS
553 timer->start_site = NULL;
554 timer->start_pid = -1;
555 memset(timer->start_comm, 0, TASK_COMM_LEN);
556#endif
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100557 lockdep_init_map(&timer->lockdep_map, name, key, 0);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700558}
559
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700560/**
Randy Dunlap633fe792009-04-01 17:47:23 -0700561 * init_timer_key - initialize a timer
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700562 * @timer: the timer to be initialized
Randy Dunlap633fe792009-04-01 17:47:23 -0700563 * @name: name of the timer
564 * @key: lockdep class key of the fake lock used for tracking timer
565 * sync lock dependencies
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700566 *
Randy Dunlap633fe792009-04-01 17:47:23 -0700567 * init_timer_key() must be done to a timer prior calling *any* of the
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700568 * other timer functions.
569 */
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100570void init_timer_key(struct timer_list *timer,
571 const char *name,
572 struct lock_class_key *key)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700573{
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800574 debug_init(timer);
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100575 __init_timer(timer, name, key);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700576}
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100577EXPORT_SYMBOL(init_timer_key);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700578
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100579void init_timer_deferrable_key(struct timer_list *timer,
580 const char *name,
581 struct lock_class_key *key)
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700582{
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100583 init_timer_key(timer, name, key);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700584 timer_set_deferrable(timer);
585}
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100586EXPORT_SYMBOL(init_timer_deferrable_key);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700587
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700588static inline void detach_timer(struct timer_list *timer,
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800589 int clear_pending)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700590{
591 struct list_head *entry = &timer->entry;
592
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800593 debug_deactivate(timer);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700594
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700595 __list_del(entry->prev, entry->next);
596 if (clear_pending)
597 entry->next = NULL;
598 entry->prev = LIST_POISON2;
599}
600
601/*
Oleg Nesterov3691c512006-03-31 02:30:30 -0800602 * We are using hashed locking: holding per_cpu(tvec_bases).lock
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700603 * means that all timers which are tied to this base via timer->base are
604 * locked, and the base itself is locked too.
605 *
606 * So __run_timers/migrate_timers can safely modify all timers which could
607 * be found on ->tvX lists.
608 *
609 * When the timer's base is locked, and the timer removed from list, it is
610 * possible to set timer->base = NULL and drop the lock: the timer remains
611 * locked.
612 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100613static struct tvec_base *lock_timer_base(struct timer_list *timer,
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700614 unsigned long *flags)
Josh Triplett89e7e3742006-09-29 01:59:36 -0700615 __acquires(timer->base->lock)
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700616{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100617 struct tvec_base *base;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700618
619 for (;;) {
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100620 struct tvec_base *prelock_base = timer->base;
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700621 base = tbase_get_base(prelock_base);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700622 if (likely(base != NULL)) {
623 spin_lock_irqsave(&base->lock, *flags);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700624 if (likely(prelock_base == timer->base))
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700625 return base;
626 /* The timer has migrated to another CPU */
627 spin_unlock_irqrestore(&base->lock, *flags);
628 }
629 cpu_relax();
630 }
631}
632
Ingo Molnar74019222009-02-18 12:23:29 +0100633static inline int
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530634__mod_timer(struct timer_list *timer, unsigned long expires,
635 bool pending_only, int pinned)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100637 struct tvec_base *base, *new_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 unsigned long flags;
Arun R Bharadwajeea08f32009-04-16 12:16:41 +0530639 int ret = 0 , cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800641 timer_stats_timer_set_start_info(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 BUG_ON(!timer->function);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700644 base = lock_timer_base(timer, &flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700646 if (timer_pending(timer)) {
647 detach_timer(timer, 0);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200648 if (timer->expires == base->next_timer &&
649 !tbase_get_deferrable(timer->base))
650 base->next_timer = base->timer_jiffies;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 ret = 1;
Ingo Molnar74019222009-02-18 12:23:29 +0100652 } else {
653 if (pending_only)
654 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 }
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700656
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800657 debug_activate(timer, expires);
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -0700658
Jan Beulicha4a61982006-03-24 03:15:54 -0800659 new_base = __get_cpu_var(tvec_bases);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700660
Arun R Bharadwajeea08f32009-04-16 12:16:41 +0530661 cpu = smp_processor_id();
662
663#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
664 if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) {
665 int preferred_cpu = get_nohz_load_balancer();
666
667 if (preferred_cpu >= 0)
668 cpu = preferred_cpu;
669 }
670#endif
671 new_base = per_cpu(tvec_bases, cpu);
672
Oleg Nesterov3691c512006-03-31 02:30:30 -0800673 if (base != new_base) {
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700674 /*
675 * We are trying to schedule the timer on the local CPU.
676 * However we can't change timer's base while it is running,
677 * otherwise del_timer_sync() can't detect that the timer's
678 * handler yet has not finished. This also guarantees that
679 * the timer is serialized wrt itself.
680 */
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800681 if (likely(base->running_timer != timer)) {
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700682 /* See the comment in lock_timer_base() */
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700683 timer_set_base(timer, NULL);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700684 spin_unlock(&base->lock);
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800685 base = new_base;
686 spin_lock(&base->lock);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700687 timer_set_base(timer, base);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700688 }
689 }
690
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 timer->expires = expires;
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200692 if (time_before(timer->expires, base->next_timer) &&
693 !tbase_get_deferrable(timer->base))
694 base->next_timer = timer->expires;
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800695 internal_add_timer(base, timer);
Ingo Molnar74019222009-02-18 12:23:29 +0100696
697out_unlock:
Oleg Nesterova2c348f2006-03-31 02:30:31 -0800698 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699
700 return ret;
701}
702
Ingo Molnar74019222009-02-18 12:23:29 +0100703/**
704 * mod_timer_pending - modify a pending timer's timeout
705 * @timer: the pending timer to be modified
706 * @expires: new timeout in jiffies
707 *
708 * mod_timer_pending() is the same for pending timers as mod_timer(),
709 * but will not re-activate and modify already deleted timers.
710 *
711 * It is useful for unserialized use of timers.
712 */
713int mod_timer_pending(struct timer_list *timer, unsigned long expires)
714{
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530715 return __mod_timer(timer, expires, true, TIMER_NOT_PINNED);
Ingo Molnar74019222009-02-18 12:23:29 +0100716}
717EXPORT_SYMBOL(mod_timer_pending);
718
719/**
720 * mod_timer - modify a timer's timeout
721 * @timer: the timer to be modified
722 * @expires: new timeout in jiffies
723 *
724 * mod_timer() is a more efficient way to update the expire field of an
725 * active timer (if the timer is inactive it will be activated)
726 *
727 * mod_timer(timer, expires) is equivalent to:
728 *
729 * del_timer(timer); timer->expires = expires; add_timer(timer);
730 *
731 * Note that if there are multiple unserialized concurrent users of the
732 * same timer, then mod_timer() is the only safe way to modify the timeout,
733 * since add_timer() cannot modify an already running timer.
734 *
735 * The function returns whether it has modified a pending timer or not.
736 * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an
737 * active timer returns 1.)
738 */
739int mod_timer(struct timer_list *timer, unsigned long expires)
740{
741 /*
742 * This is a common optimization triggered by the
743 * networking code - if the timer is re-modified
744 * to be the same thing then just return:
745 */
Pavel Roskin48411582009-07-18 16:46:02 -0400746 if (timer_pending(timer) && timer->expires == expires)
Ingo Molnar74019222009-02-18 12:23:29 +0100747 return 1;
748
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530749 return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);
Ingo Molnar74019222009-02-18 12:23:29 +0100750}
751EXPORT_SYMBOL(mod_timer);
752
753/**
Arun R Bharadwaj597d0272009-04-16 12:13:26 +0530754 * mod_timer_pinned - modify a timer's timeout
755 * @timer: the timer to be modified
756 * @expires: new timeout in jiffies
757 *
758 * mod_timer_pinned() is a way to update the expire field of an
759 * active timer (if the timer is inactive it will be activated)
760 * and not allow the timer to be migrated to a different CPU.
761 *
762 * mod_timer_pinned(timer, expires) is equivalent to:
763 *
764 * del_timer(timer); timer->expires = expires; add_timer(timer);
765 */
766int mod_timer_pinned(struct timer_list *timer, unsigned long expires)
767{
768 if (timer->expires == expires && timer_pending(timer))
769 return 1;
770
771 return __mod_timer(timer, expires, false, TIMER_PINNED);
772}
773EXPORT_SYMBOL(mod_timer_pinned);
774
775/**
Ingo Molnar74019222009-02-18 12:23:29 +0100776 * add_timer - start a timer
777 * @timer: the timer to be added
778 *
779 * The kernel will do a ->function(->data) callback from the
780 * timer interrupt at the ->expires point in the future. The
781 * current time is 'jiffies'.
782 *
783 * The timer's ->expires, ->function (and if the handler uses it, ->data)
784 * fields must be set prior calling this function.
785 *
786 * Timers with an ->expires field in the past will be executed in the next
787 * timer tick.
788 */
789void add_timer(struct timer_list *timer)
790{
791 BUG_ON(timer_pending(timer));
792 mod_timer(timer, timer->expires);
793}
794EXPORT_SYMBOL(add_timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700796/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 * add_timer_on - start a timer on a particular CPU
798 * @timer: the timer to be added
799 * @cpu: the CPU to start it on
800 *
801 * This is not very scalable on SMP. Double adds are not possible.
802 */
803void add_timer_on(struct timer_list *timer, int cpu)
804{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100805 struct tvec_base *base = per_cpu(tvec_bases, cpu);
Thomas Gleixner68194572007-07-19 01:49:16 -0700806 unsigned long flags;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700807
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800808 timer_stats_timer_set_start_info(timer);
Thomas Gleixner68194572007-07-19 01:49:16 -0700809 BUG_ON(timer_pending(timer) || !timer->function);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800810 spin_lock_irqsave(&base->lock, flags);
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700811 timer_set_base(timer, base);
Xiao Guangrong2b022e32009-08-10 10:48:59 +0800812 debug_activate(timer, timer->expires);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200813 if (time_before(timer->expires, base->next_timer) &&
814 !tbase_get_deferrable(timer->base))
815 base->next_timer = timer->expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 internal_add_timer(base, timer);
Thomas Gleixner06d83082008-03-22 09:20:24 +0100817 /*
818 * Check whether the other CPU is idle and needs to be
819 * triggered to reevaluate the timer wheel when nohz is
820 * active. We are protected against the other CPU fiddling
821 * with the timer by holding the timer base lock. This also
822 * makes sure that a CPU on the way to idle can not evaluate
823 * the timer wheel.
824 */
825 wake_up_idle_cpu(cpu);
Oleg Nesterov3691c512006-03-31 02:30:30 -0800826 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827}
Andi Kleena9862e02009-05-19 22:49:07 +0200828EXPORT_SYMBOL_GPL(add_timer_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700830/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831 * del_timer - deactive a timer.
832 * @timer: the timer to be deactivated
833 *
834 * del_timer() deactivates a timer - this works on both active and inactive
835 * timers.
836 *
837 * The function returns whether it has deactivated a pending timer or not.
838 * (ie. del_timer() of an inactive timer returns 0, del_timer() of an
839 * active timer returns 1.)
840 */
841int del_timer(struct timer_list *timer)
842{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100843 struct tvec_base *base;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844 unsigned long flags;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700845 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800847 timer_stats_timer_clear_start_info(timer);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700848 if (timer_pending(timer)) {
849 base = lock_timer_base(timer, &flags);
850 if (timer_pending(timer)) {
851 detach_timer(timer, 1);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200852 if (timer->expires == base->next_timer &&
853 !tbase_get_deferrable(timer->base))
854 base->next_timer = base->timer_jiffies;
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700855 ret = 1;
856 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857 spin_unlock_irqrestore(&base->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700860 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862EXPORT_SYMBOL(del_timer);
863
864#ifdef CONFIG_SMP
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700865/**
866 * try_to_del_timer_sync - Try to deactivate a timer
867 * @timer: timer do del
868 *
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700869 * This function tries to deactivate a timer. Upon successful (ret >= 0)
870 * exit the timer is not queued and the handler is not running on any CPU.
871 *
872 * It must not be called from interrupt contexts.
873 */
874int try_to_del_timer_sync(struct timer_list *timer)
875{
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100876 struct tvec_base *base;
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700877 unsigned long flags;
878 int ret = -1;
879
880 base = lock_timer_base(timer, &flags);
881
882 if (base->running_timer == timer)
883 goto out;
884
885 ret = 0;
886 if (timer_pending(timer)) {
887 detach_timer(timer, 1);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +0200888 if (timer->expires == base->next_timer &&
889 !tbase_get_deferrable(timer->base))
890 base->next_timer = base->timer_jiffies;
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700891 ret = 1;
892 }
893out:
894 spin_unlock_irqrestore(&base->lock, flags);
895
896 return ret;
897}
David Howellse19dff12007-04-26 15:46:56 -0700898EXPORT_SYMBOL(try_to_del_timer_sync);
899
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700900/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 * del_timer_sync - deactivate a timer and wait for the handler to finish.
902 * @timer: the timer to be deactivated
903 *
904 * This function only differs from del_timer() on SMP: besides deactivating
905 * the timer it also makes sure the handler has finished executing on other
906 * CPUs.
907 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800908 * Synchronization rules: Callers must prevent restarting of the timer,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909 * otherwise this function is meaningless. It must not be called from
910 * interrupt contexts. The caller must not hold locks which would prevent
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700911 * completion of the timer's handler. The timer's handler must not call
912 * add_timer_on(). Upon exit the timer is not queued and the handler is
913 * not running on any CPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 *
915 * The function returns whether it has deactivated a pending timer or not.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 */
917int del_timer_sync(struct timer_list *timer)
918{
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100919#ifdef CONFIG_LOCKDEP
920 unsigned long flags;
921
922 local_irq_save(flags);
923 lock_map_acquire(&timer->lockdep_map);
924 lock_map_release(&timer->lockdep_map);
925 local_irq_restore(flags);
926#endif
927
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700928 for (;;) {
929 int ret = try_to_del_timer_sync(timer);
930 if (ret >= 0)
931 return ret;
Andrew Mortona0009652006-07-14 00:24:06 -0700932 cpu_relax();
Oleg Nesterovfd450b72005-06-23 00:08:59 -0700933 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934}
935EXPORT_SYMBOL(del_timer_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700936#endif
937
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100938static int cascade(struct tvec_base *base, struct tvec *tv, int index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939{
940 /* cascade all the timers from tv up one level */
Porpoise3439dd82006-06-23 02:05:56 -0700941 struct timer_list *timer, *tmp;
942 struct list_head tv_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943
Porpoise3439dd82006-06-23 02:05:56 -0700944 list_replace_init(tv->vec + index, &tv_list);
945
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 /*
Porpoise3439dd82006-06-23 02:05:56 -0700947 * We are removing _all_ timers from the list, so we
948 * don't have to detach them individually.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949 */
Porpoise3439dd82006-06-23 02:05:56 -0700950 list_for_each_entry_safe(timer, tmp, &tv_list, entry) {
Venki Pallipadi6e453a62007-05-08 00:27:44 -0700951 BUG_ON(tbase_get_base(timer->base) != base);
Porpoise3439dd82006-06-23 02:05:56 -0700952 internal_add_timer(base, timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700954
955 return index;
956}
957
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -0700958#define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK)
959
960/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 * __run_timers - run all expired timers (if any) on this CPU.
962 * @base: the timer vector to be processed.
963 *
964 * This function cascades all vectors and executes all expired timer
965 * vectors.
966 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +0100967static inline void __run_timers(struct tvec_base *base)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968{
969 struct timer_list *timer;
970
Oleg Nesterov3691c512006-03-31 02:30:30 -0800971 spin_lock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 while (time_after_eq(jiffies, base->timer_jiffies)) {
Oleg Nesterov626ab0e2006-06-23 02:05:55 -0700973 struct list_head work_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974 struct list_head *head = &work_list;
Thomas Gleixner68194572007-07-19 01:49:16 -0700975 int index = base->timer_jiffies & TVR_MASK;
Oleg Nesterov626ab0e2006-06-23 02:05:55 -0700976
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 /*
978 * Cascade timers:
979 */
980 if (!index &&
981 (!cascade(base, &base->tv2, INDEX(0))) &&
982 (!cascade(base, &base->tv3, INDEX(1))) &&
983 !cascade(base, &base->tv4, INDEX(2)))
984 cascade(base, &base->tv5, INDEX(3));
Oleg Nesterov626ab0e2006-06-23 02:05:55 -0700985 ++base->timer_jiffies;
986 list_replace_init(base->tv1.vec + index, &work_list);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700987 while (!list_empty(head)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 void (*fn)(unsigned long);
989 unsigned long data;
990
Pavel Emelianovb5e61812007-05-08 00:30:19 -0700991 timer = list_first_entry(head, struct timer_list,entry);
Thomas Gleixner68194572007-07-19 01:49:16 -0700992 fn = timer->function;
993 data = timer->data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800995 timer_stats_account_timer(timer);
996
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 set_running_timer(base, timer);
Oleg Nesterov55c888d2005-06-23 00:08:56 -0700998 detach_timer(timer, 1);
Johannes Berg6f2b9b92009-01-29 16:03:20 +0100999
Oleg Nesterov3691c512006-03-31 02:30:30 -08001000 spin_unlock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001 {
Jesper Juhlbe5b4fb2005-06-23 00:09:09 -07001002 int preempt_count = preempt_count();
Johannes Berg6f2b9b92009-01-29 16:03:20 +01001003
1004#ifdef CONFIG_LOCKDEP
1005 /*
1006 * It is permissible to free the timer from
1007 * inside the function that is called from
1008 * it, this we need to take into account for
1009 * lockdep too. To avoid bogus "held lock
1010 * freed" warnings as well as problems when
1011 * looking into timer->lockdep_map, make a
1012 * copy and use that here.
1013 */
1014 struct lockdep_map lockdep_map =
1015 timer->lockdep_map;
1016#endif
1017 /*
1018 * Couple the lock chain with the lock chain at
1019 * del_timer_sync() by acquiring the lock_map
1020 * around the fn() call here and in
1021 * del_timer_sync().
1022 */
1023 lock_map_acquire(&lockdep_map);
1024
Xiao Guangrong2b022e32009-08-10 10:48:59 +08001025 trace_timer_expire_entry(timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026 fn(data);
Xiao Guangrong2b022e32009-08-10 10:48:59 +08001027 trace_timer_expire_exit(timer);
Johannes Berg6f2b9b92009-01-29 16:03:20 +01001028
1029 lock_map_release(&lockdep_map);
1030
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 if (preempt_count != preempt_count()) {
Pavel Machek4c9dc642008-01-30 13:30:00 +01001032 printk(KERN_ERR "huh, entered %p "
Jesper Juhlbe5b4fb2005-06-23 00:09:09 -07001033 "with preempt_count %08x, exited"
1034 " with %08x?\n",
1035 fn, preempt_count,
1036 preempt_count());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 BUG();
1038 }
1039 }
Oleg Nesterov3691c512006-03-31 02:30:30 -08001040 spin_lock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041 }
1042 }
1043 set_running_timer(base, NULL);
Oleg Nesterov3691c512006-03-31 02:30:30 -08001044 spin_unlock_irq(&base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045}
1046
Russell Kingee9c5782008-04-20 13:59:33 +01001047#ifdef CONFIG_NO_HZ
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048/*
1049 * Find out when the next timer event is due to happen. This
Randy Dunlap90cba642009-08-25 14:35:41 -07001050 * is used on S/390 to stop all activity when a CPU is idle.
1051 * This function needs to be called with interrupts disabled.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 */
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001053static unsigned long __next_timer_interrupt(struct tvec_base *base)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054{
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001055 unsigned long timer_jiffies = base->timer_jiffies;
Thomas Gleixnereaad0842007-05-29 23:47:39 +02001056 unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001057 int index, slot, array, found = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 struct timer_list *nte;
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001059 struct tvec *varray[4];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060
1061 /* Look for timer events in tv1. */
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001062 index = slot = timer_jiffies & TVR_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063 do {
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001064 list_for_each_entry(nte, base->tv1.vec + slot, entry) {
Thomas Gleixner68194572007-07-19 01:49:16 -07001065 if (tbase_get_deferrable(nte->base))
1066 continue;
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001067
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001068 found = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 expires = nte->expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001070 /* Look at the cascade bucket(s)? */
1071 if (!index || slot < index)
1072 goto cascade;
1073 return expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 }
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001075 slot = (slot + 1) & TVR_MASK;
1076 } while (slot != index);
1077
1078cascade:
1079 /* Calculate the next cascade event */
1080 if (index)
1081 timer_jiffies += TVR_SIZE - index;
1082 timer_jiffies >>= TVR_BITS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083
1084 /* Check tv2-tv5. */
1085 varray[0] = &base->tv2;
1086 varray[1] = &base->tv3;
1087 varray[2] = &base->tv4;
1088 varray[3] = &base->tv5;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001089
1090 for (array = 0; array < 4; array++) {
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001091 struct tvec *varp = varray[array];
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001092
1093 index = slot = timer_jiffies & TVN_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 do {
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001095 list_for_each_entry(nte, varp->vec + slot, entry) {
Jon Huntera0419882009-05-01 13:10:23 -07001096 if (tbase_get_deferrable(nte->base))
1097 continue;
1098
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001099 found = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 if (time_before(nte->expires, expires))
1101 expires = nte->expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001102 }
1103 /*
1104 * Do we still search for the first timer or are
1105 * we looking up the cascade buckets ?
1106 */
1107 if (found) {
1108 /* Look at the cascade bucket(s)? */
1109 if (!index || slot < index)
1110 break;
1111 return expires;
1112 }
1113 slot = (slot + 1) & TVN_MASK;
1114 } while (slot != index);
1115
1116 if (index)
1117 timer_jiffies += TVN_SIZE - index;
1118 timer_jiffies >>= TVN_BITS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119 }
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001120 return expires;
1121}
1122
1123/*
1124 * Check, if the next hrtimer event is before the next timer wheel
1125 * event:
1126 */
1127static unsigned long cmp_next_hrtimer_event(unsigned long now,
1128 unsigned long expires)
1129{
1130 ktime_t hr_delta = hrtimer_get_next_event();
1131 struct timespec tsdelta;
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001132 unsigned long delta;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001133
1134 if (hr_delta.tv64 == KTIME_MAX)
1135 return expires;
1136
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001137 /*
1138 * Expired timer available, let it expire in the next tick
1139 */
1140 if (hr_delta.tv64 <= 0)
1141 return now + 1;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001142
1143 tsdelta = ktime_to_timespec(hr_delta);
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001144 delta = timespec_to_jiffies(&tsdelta);
Thomas Gleixnereaad0842007-05-29 23:47:39 +02001145
1146 /*
1147 * Limit the delta to the max value, which is checked in
1148 * tick_nohz_stop_sched_tick():
1149 */
1150 if (delta > NEXT_TIMER_MAX_DELTA)
1151 delta = NEXT_TIMER_MAX_DELTA;
1152
Thomas Gleixner9501b6c2007-03-25 14:31:17 +02001153 /*
1154 * Take rounding errors in to account and make sure, that it
1155 * expires in the next tick. Otherwise we go into an endless
1156 * ping pong due to tick_nohz_stop_sched_tick() retriggering
1157 * the timer softirq
1158 */
1159 if (delta < 1)
1160 delta = 1;
1161 now += delta;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001162 if (time_before(now, expires))
1163 return now;
1164 return expires;
1165}
1166
1167/**
Li Zefan8dce39c2007-11-05 14:51:10 -08001168 * get_next_timer_interrupt - return the jiffy of the next pending timer
Randy Dunlap05fb6bf2007-02-28 20:12:13 -08001169 * @now: current time (in jiffies)
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001170 */
Thomas Gleixnerfd064b92007-02-16 01:27:47 -08001171unsigned long get_next_timer_interrupt(unsigned long now)
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001172{
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001173 struct tvec_base *base = __get_cpu_var(tvec_bases);
Thomas Gleixnerfd064b92007-02-16 01:27:47 -08001174 unsigned long expires;
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001175
1176 spin_lock(&base->lock);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +02001177 if (time_before_eq(base->next_timer, base->timer_jiffies))
1178 base->next_timer = __next_timer_interrupt(base);
1179 expires = base->next_timer;
Oleg Nesterov3691c512006-03-31 02:30:30 -08001180 spin_unlock(&base->lock);
Tony Lindgren69239742006-03-06 15:42:45 -08001181
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001182 if (time_before_eq(expires, now))
1183 return now;
Zachary Amsden0662b712006-05-20 15:00:24 -07001184
Thomas Gleixner1cfd6842007-02-16 01:27:46 -08001185 return cmp_next_hrtimer_event(now, expires);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186}
1187#endif
1188
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189/*
Daniel Walker5b4db0c2007-10-18 03:06:11 -07001190 * Called from the timer interrupt handler to charge one tick to the current
Linus Torvalds1da177e2005-04-16 15:20:36 -07001191 * process. user_tick is 1 if the tick is user time, 0 for system.
1192 */
1193void update_process_times(int user_tick)
1194{
1195 struct task_struct *p = current;
1196 int cpu = smp_processor_id();
1197
1198 /* Note: this timer irq context must be accounted for as well. */
Paul Mackerrasfa13a5a2007-11-09 22:39:38 +01001199 account_process_tick(p, user_tick);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200 run_local_timers();
Paul E. McKenneya1572292009-08-22 13:56:51 -07001201 rcu_check_callbacks(cpu, user_tick);
Peter Zijlstrab845b512008-08-08 21:47:09 +02001202 printk_tick();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203 scheduler_tick();
Thomas Gleixner68194572007-07-19 01:49:16 -07001204 run_posix_cpu_timers(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205}
1206
1207/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 * This function runs timers and the timer-tq in bottom half context.
1209 */
1210static void run_timer_softirq(struct softirq_action *h)
1211{
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001212 struct tvec_base *base = __get_cpu_var(tvec_bases);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213
Ingo Molnarcdd6c482009-09-21 12:02:48 +02001214 perf_event_do_pending();
Peter Zijlstra925d5192009-03-30 19:07:02 +02001215
Peter Zijlstrad3d74452008-01-25 21:08:31 +01001216 hrtimer_run_pending();
Ingo Molnar82f67cd2007-02-16 01:28:13 -08001217
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218 if (time_after_eq(jiffies, base->timer_jiffies))
1219 __run_timers(base);
1220}
1221
1222/*
1223 * Called by the local, per-CPU timer interrupt on SMP.
1224 */
1225void run_local_timers(void)
1226{
Peter Zijlstrad3d74452008-01-25 21:08:31 +01001227 hrtimer_run_queues();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 raise_softirq(TIMER_SOFTIRQ);
Ingo Molnar6687a972006-03-24 03:18:41 -08001229 softlockup_tick();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230}
1231
1232/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233 * The 64-bit jiffies value is not atomic - you MUST NOT read it
1234 * without sampling the sequence number in xtime_lock.
1235 * jiffies is defined in the linker script...
1236 */
1237
Atsushi Nemoto3171a032006-09-29 02:00:32 -07001238void do_timer(unsigned long ticks)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239{
Atsushi Nemoto3171a032006-09-29 02:00:32 -07001240 jiffies_64 += ticks;
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02001241 update_wall_time();
1242 calc_global_load();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243}
1244
1245#ifdef __ARCH_WANT_SYS_ALARM
1246
1247/*
1248 * For backwards compatibility? This can be done in libc so Alpha
1249 * and all newer ports shouldn't need it.
1250 */
Heiko Carstens58fd3aa2009-01-14 14:14:03 +01001251SYSCALL_DEFINE1(alarm, unsigned int, seconds)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252{
Thomas Gleixnerc08b8a42006-03-25 03:06:33 -08001253 return alarm_setitimer(seconds);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254}
1255
1256#endif
1257
1258#ifndef __alpha__
1259
1260/*
1261 * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this
1262 * should be moved into arch/i386 instead?
1263 */
1264
1265/**
1266 * sys_getpid - return the thread group id of the current process
1267 *
1268 * Note, despite the name, this returns the tgid not the pid. The tgid and
1269 * the pid are identical unless CLONE_THREAD was specified on clone() in
1270 * which case the tgid is the same in all threads of the same group.
1271 *
1272 * This is SMP safe as current->tgid does not change.
1273 */
Heiko Carstens58fd3aa2009-01-14 14:14:03 +01001274SYSCALL_DEFINE0(getpid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001275{
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001276 return task_tgid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001277}
1278
1279/*
Kirill Korotaev6997a6f2006-08-13 23:24:23 -07001280 * Accessing ->real_parent is not SMP-safe, it could
1281 * change from under us. However, we can use a stale
1282 * value of ->real_parent under rcu_read_lock(), see
1283 * release_task()->call_rcu(delayed_put_task_struct).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001284 */
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001285SYSCALL_DEFINE0(getppid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286{
1287 int pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288
Kirill Korotaev6997a6f2006-08-13 23:24:23 -07001289 rcu_read_lock();
Pavel Emelyanov6c5f3e72008-02-08 04:19:20 -08001290 pid = task_tgid_vnr(current->real_parent);
Kirill Korotaev6997a6f2006-08-13 23:24:23 -07001291 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292
Linus Torvalds1da177e2005-04-16 15:20:36 -07001293 return pid;
1294}
1295
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001296SYSCALL_DEFINE0(getuid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297{
1298 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001299 return current_uid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300}
1301
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001302SYSCALL_DEFINE0(geteuid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303{
1304 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001305 return current_euid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306}
1307
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001308SYSCALL_DEFINE0(getgid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309{
1310 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001311 return current_gid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312}
1313
Heiko Carstensdbf040d2009-01-14 14:14:04 +01001314SYSCALL_DEFINE0(getegid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315{
1316 /* Only we change this so SMP safe */
David Howells76aac0e2008-11-14 10:39:12 +11001317 return current_egid();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001318}
1319
1320#endif
1321
1322static void process_timeout(unsigned long __data)
1323{
Ingo Molnar36c8b582006-07-03 00:25:41 -07001324 wake_up_process((struct task_struct *)__data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325}
1326
1327/**
1328 * schedule_timeout - sleep until timeout
1329 * @timeout: timeout value in jiffies
1330 *
1331 * Make the current task sleep until @timeout jiffies have
1332 * elapsed. The routine will return immediately unless
1333 * the current task state has been set (see set_current_state()).
1334 *
1335 * You can set the task state as follows -
1336 *
1337 * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
1338 * pass before the routine returns. The routine will return 0
1339 *
1340 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1341 * delivered to the current task. In this case the remaining time
1342 * in jiffies will be returned, or 0 if the timer expired in time
1343 *
1344 * The current task state is guaranteed to be TASK_RUNNING when this
1345 * routine returns.
1346 *
1347 * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
1348 * the CPU away without a bound on the timeout. In this case the return
1349 * value will be %MAX_SCHEDULE_TIMEOUT.
1350 *
1351 * In all cases the return value is guaranteed to be non-negative.
1352 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08001353signed long __sched schedule_timeout(signed long timeout)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354{
1355 struct timer_list timer;
1356 unsigned long expire;
1357
1358 switch (timeout)
1359 {
1360 case MAX_SCHEDULE_TIMEOUT:
1361 /*
1362 * These two special cases are useful to be comfortable
1363 * in the caller. Nothing more. We could take
1364 * MAX_SCHEDULE_TIMEOUT from one of the negative value
1365 * but I' d like to return a valid offset (>=0) to allow
1366 * the caller to do everything it want with the retval.
1367 */
1368 schedule();
1369 goto out;
1370 default:
1371 /*
1372 * Another bit of PARANOID. Note that the retval will be
1373 * 0 since no piece of kernel is supposed to do a check
1374 * for a negative retval of schedule_timeout() (since it
1375 * should never happens anyway). You just have the printk()
1376 * that will tell you if something is gone wrong and where.
1377 */
Andrew Morton5b149bc2006-12-22 01:10:14 -08001378 if (timeout < 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 printk(KERN_ERR "schedule_timeout: wrong timeout "
Andrew Morton5b149bc2006-12-22 01:10:14 -08001380 "value %lx\n", timeout);
1381 dump_stack();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382 current->state = TASK_RUNNING;
1383 goto out;
1384 }
1385 }
1386
1387 expire = timeout + jiffies;
1388
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -07001389 setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);
Arun R Bharadwaj597d0272009-04-16 12:13:26 +05301390 __mod_timer(&timer, expire, false, TIMER_NOT_PINNED);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391 schedule();
1392 del_singleshot_timer_sync(&timer);
1393
Thomas Gleixnerc6f3a972008-04-30 00:55:03 -07001394 /* Remove the timer from the object tracker */
1395 destroy_timer_on_stack(&timer);
1396
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397 timeout = expire - jiffies;
1398
1399 out:
1400 return timeout < 0 ? 0 : timeout;
1401}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402EXPORT_SYMBOL(schedule_timeout);
1403
Andrew Morton8a1c1752005-09-13 01:25:15 -07001404/*
1405 * We can use __set_current_state() here because schedule_timeout() calls
1406 * schedule() unconditionally.
1407 */
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001408signed long __sched schedule_timeout_interruptible(signed long timeout)
1409{
Andrew Mortona5a0d522005-10-30 15:01:42 -08001410 __set_current_state(TASK_INTERRUPTIBLE);
1411 return schedule_timeout(timeout);
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001412}
1413EXPORT_SYMBOL(schedule_timeout_interruptible);
1414
Matthew Wilcox294d5cc2007-12-06 11:59:46 -05001415signed long __sched schedule_timeout_killable(signed long timeout)
1416{
1417 __set_current_state(TASK_KILLABLE);
1418 return schedule_timeout(timeout);
1419}
1420EXPORT_SYMBOL(schedule_timeout_killable);
1421
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001422signed long __sched schedule_timeout_uninterruptible(signed long timeout)
1423{
Andrew Mortona5a0d522005-10-30 15:01:42 -08001424 __set_current_state(TASK_UNINTERRUPTIBLE);
1425 return schedule_timeout(timeout);
Nishanth Aravamudan64ed93a2005-09-10 00:27:21 -07001426}
1427EXPORT_SYMBOL(schedule_timeout_uninterruptible);
1428
Linus Torvalds1da177e2005-04-16 15:20:36 -07001429/* Thread ID - the internal kernel "pid" */
Heiko Carstens58fd3aa2009-01-14 14:14:03 +01001430SYSCALL_DEFINE0(gettid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431{
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001432 return task_pid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433}
1434
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -07001435/**
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001436 * do_sysinfo - fill in sysinfo struct
Rolf Eike Beer2aae4a12006-09-29 01:59:46 -07001437 * @info: pointer to buffer to fill
Thomas Gleixner68194572007-07-19 01:49:16 -07001438 */
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001439int do_sysinfo(struct sysinfo *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001440{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441 unsigned long mem_total, sav_total;
1442 unsigned int mem_unit, bitcount;
Thomas Gleixner2d024942009-05-02 20:08:52 +02001443 struct timespec tp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001444
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001445 memset(info, 0, sizeof(struct sysinfo));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001446
Thomas Gleixner2d024942009-05-02 20:08:52 +02001447 ktime_get_ts(&tp);
1448 monotonic_to_bootbased(&tp);
1449 info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450
Thomas Gleixner2d024942009-05-02 20:08:52 +02001451 get_avenrun(info->loads, 0, SI_LOAD_SHIFT - FSHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452
Thomas Gleixner2d024942009-05-02 20:08:52 +02001453 info->procs = nr_threads;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001455 si_meminfo(info);
1456 si_swapinfo(info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457
1458 /*
1459 * If the sum of all the available memory (i.e. ram + swap)
1460 * is less than can be stored in a 32 bit unsigned long then
1461 * we can be binary compatible with 2.2.x kernels. If not,
1462 * well, in that case 2.2.x was broken anyways...
1463 *
1464 * -Erik Andersen <andersee@debian.org>
1465 */
1466
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001467 mem_total = info->totalram + info->totalswap;
1468 if (mem_total < info->totalram || mem_total < info->totalswap)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001469 goto out;
1470 bitcount = 0;
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001471 mem_unit = info->mem_unit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472 while (mem_unit > 1) {
1473 bitcount++;
1474 mem_unit >>= 1;
1475 sav_total = mem_total;
1476 mem_total <<= 1;
1477 if (mem_total < sav_total)
1478 goto out;
1479 }
1480
1481 /*
1482 * If mem_total did not overflow, multiply all memory values by
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001483 * info->mem_unit and set it to 1. This leaves things compatible
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 * with 2.2.x, and also retains compatibility with earlier 2.4.x
1485 * kernels...
1486 */
1487
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001488 info->mem_unit = 1;
1489 info->totalram <<= bitcount;
1490 info->freeram <<= bitcount;
1491 info->sharedram <<= bitcount;
1492 info->bufferram <<= bitcount;
1493 info->totalswap <<= bitcount;
1494 info->freeswap <<= bitcount;
1495 info->totalhigh <<= bitcount;
1496 info->freehigh <<= bitcount;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001498out:
1499 return 0;
1500}
1501
Heiko Carstens1e7bfb22009-01-14 14:14:29 +01001502SYSCALL_DEFINE1(sysinfo, struct sysinfo __user *, info)
Kyle McMartind4d23ad2007-02-10 01:46:00 -08001503{
1504 struct sysinfo val;
1505
1506 do_sysinfo(&val);
1507
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508 if (copy_to_user(info, &val, sizeof(struct sysinfo)))
1509 return -EFAULT;
1510
1511 return 0;
1512}
1513
Adrian Bunkb4be6252007-12-18 18:05:58 +01001514static int __cpuinit init_timers_cpu(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515{
1516 int j;
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001517 struct tvec_base *base;
Adrian Bunkb4be6252007-12-18 18:05:58 +01001518 static char __cpuinitdata tvec_base_done[NR_CPUS];
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001519
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001520 if (!tvec_base_done[cpu]) {
Jan Beulicha4a61982006-03-24 03:15:54 -08001521 static char boot_done;
1522
Jan Beulicha4a61982006-03-24 03:15:54 -08001523 if (boot_done) {
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001524 /*
1525 * The APs use this path later in boot
1526 */
Christoph Lameter94f60302007-07-17 04:03:29 -07001527 base = kmalloc_node(sizeof(*base),
1528 GFP_KERNEL | __GFP_ZERO,
Jan Beulicha4a61982006-03-24 03:15:54 -08001529 cpu_to_node(cpu));
1530 if (!base)
1531 return -ENOMEM;
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001532
1533 /* Make sure that tvec_base is 2 byte aligned */
1534 if (tbase_get_deferrable(base)) {
1535 WARN_ON(1);
1536 kfree(base);
1537 return -ENOMEM;
1538 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001539 per_cpu(tvec_bases, cpu) = base;
Jan Beulicha4a61982006-03-24 03:15:54 -08001540 } else {
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001541 /*
1542 * This is for the boot CPU - we use compile-time
1543 * static initialisation because per-cpu memory isn't
1544 * ready yet and because the memory allocators are not
1545 * initialised either.
1546 */
Jan Beulicha4a61982006-03-24 03:15:54 -08001547 boot_done = 1;
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001548 base = &boot_tvec_bases;
Jan Beulicha4a61982006-03-24 03:15:54 -08001549 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001550 tvec_base_done[cpu] = 1;
1551 } else {
1552 base = per_cpu(tvec_bases, cpu);
Jan Beulicha4a61982006-03-24 03:15:54 -08001553 }
Andrew Mortonba6edfc2006-04-10 22:53:58 -07001554
Oleg Nesterov3691c512006-03-31 02:30:30 -08001555 spin_lock_init(&base->lock);
Ingo Molnard730e882006-07-03 00:25:10 -07001556
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 for (j = 0; j < TVN_SIZE; j++) {
1558 INIT_LIST_HEAD(base->tv5.vec + j);
1559 INIT_LIST_HEAD(base->tv4.vec + j);
1560 INIT_LIST_HEAD(base->tv3.vec + j);
1561 INIT_LIST_HEAD(base->tv2.vec + j);
1562 }
1563 for (j = 0; j < TVR_SIZE; j++)
1564 INIT_LIST_HEAD(base->tv1.vec + j);
1565
1566 base->timer_jiffies = jiffies;
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +02001567 base->next_timer = base->timer_jiffies;
Jan Beulicha4a61982006-03-24 03:15:54 -08001568 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569}
1570
1571#ifdef CONFIG_HOTPLUG_CPU
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001572static void migrate_timer_list(struct tvec_base *new_base, struct list_head *head)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573{
1574 struct timer_list *timer;
1575
1576 while (!list_empty(head)) {
Pavel Emelianovb5e61812007-05-08 00:30:19 -07001577 timer = list_first_entry(head, struct timer_list, entry);
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001578 detach_timer(timer, 0);
Venki Pallipadi6e453a62007-05-08 00:27:44 -07001579 timer_set_base(timer, new_base);
Martin Schwidefsky97fd9ed2009-07-21 20:25:05 +02001580 if (time_before(timer->expires, new_base->next_timer) &&
1581 !tbase_get_deferrable(timer->base))
1582 new_base->next_timer = timer->expires;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 internal_add_timer(new_base, timer);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001584 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585}
1586
Randy Dunlap48ccf3d2008-01-21 17:18:25 -08001587static void __cpuinit migrate_timers(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588{
Pavel Macheka6fa8e52008-01-30 13:30:00 +01001589 struct tvec_base *old_base;
1590 struct tvec_base *new_base;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001591 int i;
1592
1593 BUG_ON(cpu_online(cpu));
Jan Beulicha4a61982006-03-24 03:15:54 -08001594 old_base = per_cpu(tvec_bases, cpu);
1595 new_base = get_cpu_var(tvec_bases);
Oleg Nesterovd82f0b02008-08-20 16:46:04 -07001596 /*
1597 * The caller is globally serialized and nobody else
1598 * takes two locks at once, deadlock is not possible.
1599 */
1600 spin_lock_irq(&new_base->lock);
Oleg Nesterov0d180402008-04-04 20:54:10 +02001601 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001602
Oleg Nesterov3691c512006-03-31 02:30:30 -08001603 BUG_ON(old_base->running_timer);
1604
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605 for (i = 0; i < TVR_SIZE; i++)
Oleg Nesterov55c888d2005-06-23 00:08:56 -07001606 migrate_timer_list(new_base, old_base->tv1.vec + i);
1607 for (i = 0; i < TVN_SIZE; i++) {
1608 migrate_timer_list(new_base, old_base->tv2.vec + i);
1609 migrate_timer_list(new_base, old_base->tv3.vec + i);
1610 migrate_timer_list(new_base, old_base->tv4.vec + i);
1611 migrate_timer_list(new_base, old_base->tv5.vec + i);
1612 }
1613
Oleg Nesterov0d180402008-04-04 20:54:10 +02001614 spin_unlock(&old_base->lock);
Oleg Nesterovd82f0b02008-08-20 16:46:04 -07001615 spin_unlock_irq(&new_base->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001616 put_cpu_var(tvec_bases);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617}
1618#endif /* CONFIG_HOTPLUG_CPU */
1619
Chandra Seetharaman8c78f302006-07-30 03:03:35 -07001620static int __cpuinit timer_cpu_notify(struct notifier_block *self,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001621 unsigned long action, void *hcpu)
1622{
1623 long cpu = (long)hcpu;
1624 switch(action) {
1625 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001626 case CPU_UP_PREPARE_FROZEN:
Jan Beulicha4a61982006-03-24 03:15:54 -08001627 if (init_timers_cpu(cpu) < 0)
1628 return NOTIFY_BAD;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629 break;
1630#ifdef CONFIG_HOTPLUG_CPU
1631 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001632 case CPU_DEAD_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 migrate_timers(cpu);
1634 break;
1635#endif
1636 default:
1637 break;
1638 }
1639 return NOTIFY_OK;
1640}
1641
Chandra Seetharaman8c78f302006-07-30 03:03:35 -07001642static struct notifier_block __cpuinitdata timers_nb = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643 .notifier_call = timer_cpu_notify,
1644};
1645
1646
1647void __init init_timers(void)
1648{
Akinobu Mita07dccf32006-09-29 02:00:22 -07001649 int err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001650 (void *)(long)smp_processor_id());
Akinobu Mita07dccf32006-09-29 02:00:22 -07001651
Ingo Molnar82f67cd2007-02-16 01:28:13 -08001652 init_timer_stats();
1653
Akinobu Mita07dccf32006-09-29 02:00:22 -07001654 BUG_ON(err == NOTIFY_BAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001655 register_cpu_notifier(&timers_nb);
Carlos R. Mafra962cf362008-05-15 11:15:37 -03001656 open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657}
1658
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659/**
1660 * msleep - sleep safely even with waitqueue interruptions
1661 * @msecs: Time in milliseconds to sleep for
1662 */
1663void msleep(unsigned int msecs)
1664{
1665 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1666
Nishanth Aravamudan75bcc8c2005-09-10 00:27:24 -07001667 while (timeout)
1668 timeout = schedule_timeout_uninterruptible(timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669}
1670
1671EXPORT_SYMBOL(msleep);
1672
1673/**
Domen Puncer96ec3ef2005-06-25 14:58:43 -07001674 * msleep_interruptible - sleep waiting for signals
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675 * @msecs: Time in milliseconds to sleep for
1676 */
1677unsigned long msleep_interruptible(unsigned int msecs)
1678{
1679 unsigned long timeout = msecs_to_jiffies(msecs) + 1;
1680
Nishanth Aravamudan75bcc8c2005-09-10 00:27:24 -07001681 while (timeout && !signal_pending(current))
1682 timeout = schedule_timeout_interruptible(timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001683 return jiffies_to_msecs(timeout);
1684}
1685
1686EXPORT_SYMBOL(msleep_interruptible);