blob: 80666f6cd4f9cd5caf55b633d369dd78423c1f6a [file] [log] [blame]
Thomas Gleixnerc0a31322006-01-09 20:52:32 -08001/*
2 * linux/kernel/hrtimer.c
3 *
4 * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
6 *
7 * High-resolution kernel timers
8 *
9 * In contrast to the low-resolution timeout API implemented in
10 * kernel/timer.c, hrtimers provide finer resolution and accuracy
11 * depending on system configuration and capabilities.
12 *
13 * These timers are currently used for:
14 * - itimers
15 * - POSIX timers
16 * - nanosleep
17 * - precise in-kernel timing
18 *
19 * Started by: Thomas Gleixner and Ingo Molnar
20 *
21 * Credits:
22 * based on kernel/timer.c
23 *
Thomas Gleixner66188fa2006-02-01 03:05:13 -080024 * Help, testing, suggestions, bugfixes, improvements were
25 * provided by:
26 *
27 * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
28 * et. al.
29 *
Thomas Gleixnerc0a31322006-01-09 20:52:32 -080030 * For licencing details see kernel-base/COPYING
31 */
32
33#include <linux/cpu.h>
34#include <linux/module.h>
35#include <linux/percpu.h>
36#include <linux/hrtimer.h>
37#include <linux/notifier.h>
38#include <linux/syscalls.h>
39#include <linux/interrupt.h>
40
41#include <asm/uaccess.h>
42
43/**
44 * ktime_get - get the monotonic time in ktime_t format
45 *
46 * returns the time in ktime_t format
47 */
48static ktime_t ktime_get(void)
49{
50 struct timespec now;
51
52 ktime_get_ts(&now);
53
54 return timespec_to_ktime(now);
55}
56
57/**
58 * ktime_get_real - get the real (wall-) time in ktime_t format
59 *
60 * returns the time in ktime_t format
61 */
62static ktime_t ktime_get_real(void)
63{
64 struct timespec now;
65
66 getnstimeofday(&now);
67
68 return timespec_to_ktime(now);
69}
70
71EXPORT_SYMBOL_GPL(ktime_get_real);
72
73/*
74 * The timer bases:
George Anzinger7978672c2006-02-01 03:05:11 -080075 *
76 * Note: If we want to add new timer bases, we have to skip the two
77 * clock ids captured by the cpu-timers. We do this by holding empty
78 * entries rather than doing math adjustment of the clock ids.
79 * This ensures that we capture erroneous accesses to these clock ids
80 * rather than moving them into the range of valid clock id's.
Thomas Gleixnerc0a31322006-01-09 20:52:32 -080081 */
82
83#define MAX_HRTIMER_BASES 2
84
85static DEFINE_PER_CPU(struct hrtimer_base, hrtimer_bases[MAX_HRTIMER_BASES]) =
86{
87 {
88 .index = CLOCK_REALTIME,
89 .get_time = &ktime_get_real,
90 .resolution = KTIME_REALTIME_RES,
91 },
92 {
93 .index = CLOCK_MONOTONIC,
94 .get_time = &ktime_get,
95 .resolution = KTIME_MONOTONIC_RES,
96 },
97};
98
99/**
100 * ktime_get_ts - get the monotonic clock in timespec format
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800101 * @ts: pointer to timespec variable
102 *
103 * The function calculates the monotonic clock from the realtime
104 * clock and the wall_to_monotonic offset and stores the result
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800105 * in normalized timespec format in the variable pointed to by @ts.
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800106 */
107void ktime_get_ts(struct timespec *ts)
108{
109 struct timespec tomono;
110 unsigned long seq;
111
112 do {
113 seq = read_seqbegin(&xtime_lock);
114 getnstimeofday(ts);
115 tomono = wall_to_monotonic;
116
117 } while (read_seqretry(&xtime_lock, seq));
118
119 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
120 ts->tv_nsec + tomono.tv_nsec);
121}
Matt Helsley69778e32006-01-09 20:52:39 -0800122EXPORT_SYMBOL_GPL(ktime_get_ts);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800123
124/*
Thomas Gleixner92127c72006-03-26 01:38:05 -0800125 * Get the coarse grained time at the softirq based on xtime and
126 * wall_to_monotonic.
127 */
128static void hrtimer_get_softirq_time(struct hrtimer_base *base)
129{
130 ktime_t xtim, tomono;
john stultzf4304ab2007-02-16 01:27:26 -0800131 struct timespec xts;
Thomas Gleixner92127c72006-03-26 01:38:05 -0800132 unsigned long seq;
133
134 do {
135 seq = read_seqbegin(&xtime_lock);
john stultzf4304ab2007-02-16 01:27:26 -0800136#ifdef CONFIG_NO_HZ
137 getnstimeofday(&xts);
138#else
139 xts = xtime;
140#endif
Thomas Gleixner92127c72006-03-26 01:38:05 -0800141 } while (read_seqretry(&xtime_lock, seq));
142
john stultzf4304ab2007-02-16 01:27:26 -0800143 xtim = timespec_to_ktime(xts);
144 tomono = timespec_to_ktime(wall_to_monotonic);
Thomas Gleixner92127c72006-03-26 01:38:05 -0800145 base[CLOCK_REALTIME].softirq_time = xtim;
146 base[CLOCK_MONOTONIC].softirq_time = ktime_add(xtim, tomono);
147}
148
149/*
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800150 * Functions and macros which are different for UP/SMP systems are kept in a
151 * single place
152 */
153#ifdef CONFIG_SMP
154
155#define set_curr_timer(b, t) do { (b)->curr_timer = (t); } while (0)
156
157/*
158 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
159 * means that all timers which are tied to this base via timer->base are
160 * locked, and the base itself is locked too.
161 *
162 * So __run_timers/migrate_timers can safely modify all timers which could
163 * be found on the lists/queues.
164 *
165 * When the timer's base is locked, and the timer removed from list, it is
166 * possible to set timer->base = NULL and drop the lock: the timer remains
167 * locked.
168 */
169static struct hrtimer_base *lock_hrtimer_base(const struct hrtimer *timer,
170 unsigned long *flags)
171{
172 struct hrtimer_base *base;
173
174 for (;;) {
175 base = timer->base;
176 if (likely(base != NULL)) {
177 spin_lock_irqsave(&base->lock, *flags);
178 if (likely(base == timer->base))
179 return base;
180 /* The timer has migrated to another CPU: */
181 spin_unlock_irqrestore(&base->lock, *flags);
182 }
183 cpu_relax();
184 }
185}
186
187/*
188 * Switch the timer base to the current CPU when possible.
189 */
190static inline struct hrtimer_base *
191switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_base *base)
192{
193 struct hrtimer_base *new_base;
194
Jan Blunck3773dc92006-08-13 23:24:19 -0700195 new_base = &__get_cpu_var(hrtimer_bases)[base->index];
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800196
197 if (base != new_base) {
198 /*
199 * We are trying to schedule the timer on the local CPU.
200 * However we can't change timer's base while it is running,
201 * so we keep it on the same CPU. No hassle vs. reprogramming
202 * the event source in the high resolution case. The softirq
203 * code will take care of this when the timer function has
204 * completed. There is no conflict as we hold the lock until
205 * the timer is enqueued.
206 */
207 if (unlikely(base->curr_timer == timer))
208 return base;
209
210 /* See the comment in lock_timer_base() */
211 timer->base = NULL;
212 spin_unlock(&base->lock);
213 spin_lock(&new_base->lock);
214 timer->base = new_base;
215 }
216 return new_base;
217}
218
219#else /* CONFIG_SMP */
220
221#define set_curr_timer(b, t) do { } while (0)
222
223static inline struct hrtimer_base *
224lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
225{
226 struct hrtimer_base *base = timer->base;
227
228 spin_lock_irqsave(&base->lock, *flags);
229
230 return base;
231}
232
233#define switch_hrtimer_base(t, b) (b)
234
235#endif /* !CONFIG_SMP */
236
237/*
238 * Functions for the union type storage format of ktime_t which are
239 * too large for inlining:
240 */
241#if BITS_PER_LONG < 64
242# ifndef CONFIG_KTIME_SCALAR
243/**
244 * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800245 * @kt: addend
246 * @nsec: the scalar nsec value to add
247 *
248 * Returns the sum of kt and nsec in ktime_t format
249 */
250ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
251{
252 ktime_t tmp;
253
254 if (likely(nsec < NSEC_PER_SEC)) {
255 tmp.tv64 = nsec;
256 } else {
257 unsigned long rem = do_div(nsec, NSEC_PER_SEC);
258
259 tmp = ktime_set((long)nsec, rem);
260 }
261
262 return ktime_add(kt, tmp);
263}
264
265#else /* CONFIG_KTIME_SCALAR */
266
267# endif /* !CONFIG_KTIME_SCALAR */
268
269/*
270 * Divide a ktime value by a nanosecond value
271 */
Roman Zippeldf869b62006-03-26 01:38:11 -0800272static unsigned long ktime_divns(const ktime_t kt, s64 div)
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800273{
274 u64 dclc, inc, dns;
275 int sft = 0;
276
277 dclc = dns = ktime_to_ns(kt);
278 inc = div;
279 /* Make sure the divisor is less than 2^32: */
280 while (div >> 32) {
281 sft++;
282 div >>= 1;
283 }
284 dclc >>= sft;
285 do_div(dclc, (unsigned long) div);
286
287 return (unsigned long) dclc;
288}
289
290#else /* BITS_PER_LONG < 64 */
291# define ktime_divns(kt, div) (unsigned long)((kt).tv64 / (div))
292#endif /* BITS_PER_LONG >= 64 */
293
294/*
295 * Counterpart to lock_timer_base above:
296 */
297static inline
298void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
299{
300 spin_unlock_irqrestore(&timer->base->lock, *flags);
301}
302
303/**
304 * hrtimer_forward - forward the timer expiry
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800305 * @timer: hrtimer to forward
Roman Zippel44f21472006-03-26 01:38:06 -0800306 * @now: forward past this time
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800307 * @interval: the interval to forward
308 *
309 * Forward the timer expiry so it will expire in the future.
Jonathan Corbet8dca6f32006-01-16 15:58:55 -0700310 * Returns the number of overruns.
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800311 */
312unsigned long
Roman Zippel44f21472006-03-26 01:38:06 -0800313hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800314{
315 unsigned long orun = 1;
Roman Zippel44f21472006-03-26 01:38:06 -0800316 ktime_t delta;
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800317
318 delta = ktime_sub(now, timer->expires);
319
320 if (delta.tv64 < 0)
321 return 0;
322
Thomas Gleixnerc9db4fa2006-01-12 11:47:34 +0100323 if (interval.tv64 < timer->base->resolution.tv64)
324 interval.tv64 = timer->base->resolution.tv64;
325
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800326 if (unlikely(delta.tv64 >= interval.tv64)) {
Roman Zippeldf869b62006-03-26 01:38:11 -0800327 s64 incr = ktime_to_ns(interval);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800328
329 orun = ktime_divns(delta, incr);
330 timer->expires = ktime_add_ns(timer->expires, incr * orun);
331 if (timer->expires.tv64 > now.tv64)
332 return orun;
333 /*
334 * This (and the ktime_add() below) is the
335 * correction for exact:
336 */
337 orun++;
338 }
339 timer->expires = ktime_add(timer->expires, interval);
340
341 return orun;
342}
343
344/*
345 * enqueue_hrtimer - internal function to (re)start a timer
346 *
347 * The timer is inserted in expiry order. Insertion into the
348 * red black tree is O(log(n)). Must hold the base lock.
349 */
350static void enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
351{
352 struct rb_node **link = &base->active.rb_node;
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800353 struct rb_node *parent = NULL;
354 struct hrtimer *entry;
355
356 /*
357 * Find the right place in the rbtree:
358 */
359 while (*link) {
360 parent = *link;
361 entry = rb_entry(parent, struct hrtimer, node);
362 /*
363 * We dont care about collisions. Nodes with
364 * the same expiry time stay together.
365 */
366 if (timer->expires.tv64 < entry->expires.tv64)
367 link = &(*link)->rb_left;
Thomas Gleixner288867e2006-01-12 11:25:54 +0100368 else
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800369 link = &(*link)->rb_right;
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800370 }
371
372 /*
Thomas Gleixner288867e2006-01-12 11:25:54 +0100373 * Insert the timer to the rbtree and check whether it
374 * replaces the first pending timer
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800375 */
376 rb_link_node(&timer->node, parent, link);
377 rb_insert_color(&timer->node, &base->active);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800378
Thomas Gleixner288867e2006-01-12 11:25:54 +0100379 if (!base->first || timer->expires.tv64 <
380 rb_entry(base->first, struct hrtimer, node)->expires.tv64)
381 base->first = &timer->node;
382}
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800383
384/*
385 * __remove_hrtimer - internal function to remove a timer
386 *
387 * Caller must hold the base lock.
388 */
389static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
390{
391 /*
Thomas Gleixner288867e2006-01-12 11:25:54 +0100392 * Remove the timer from the rbtree and replace the
393 * first entry pointer if necessary.
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800394 */
Thomas Gleixner288867e2006-01-12 11:25:54 +0100395 if (base->first == &timer->node)
396 base->first = rb_next(&timer->node);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800397 rb_erase(&timer->node, &base->active);
David Woodhouseed198cb2006-04-22 02:38:50 +0100398 rb_set_parent(&timer->node, &timer->node);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800399}
400
401/*
402 * remove hrtimer, called with base lock held
403 */
404static inline int
405remove_hrtimer(struct hrtimer *timer, struct hrtimer_base *base)
406{
407 if (hrtimer_active(timer)) {
408 __remove_hrtimer(timer, base);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800409 return 1;
410 }
411 return 0;
412}
413
414/**
415 * hrtimer_start - (re)start an relative timer on the current CPU
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800416 * @timer: the timer to be added
417 * @tim: expiry time
418 * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
419 *
420 * Returns:
421 * 0 on success
422 * 1 when the timer was active
423 */
424int
425hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
426{
427 struct hrtimer_base *base, *new_base;
428 unsigned long flags;
429 int ret;
430
431 base = lock_hrtimer_base(timer, &flags);
432
433 /* Remove an active timer from the queue: */
434 ret = remove_hrtimer(timer, base);
435
436 /* Switch the timer base, if necessary: */
437 new_base = switch_hrtimer_base(timer, base);
438
Ingo Molnar06027bd2006-02-14 13:53:15 -0800439 if (mode == HRTIMER_REL) {
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800440 tim = ktime_add(tim, new_base->get_time());
Ingo Molnar06027bd2006-02-14 13:53:15 -0800441 /*
442 * CONFIG_TIME_LOW_RES is a temporary way for architectures
443 * to signal that they simply return xtime in
444 * do_gettimeoffset(). In this case we want to round up by
445 * resolution when starting a relative timer, to avoid short
446 * timeouts. This will go away with the GTOD framework.
447 */
448#ifdef CONFIG_TIME_LOW_RES
449 tim = ktime_add(tim, base->resolution);
450#endif
451 }
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800452 timer->expires = tim;
453
454 enqueue_hrtimer(timer, new_base);
455
456 unlock_hrtimer_base(timer, &flags);
457
458 return ret;
459}
Stephen Hemminger8d16b762006-05-30 21:26:09 -0700460EXPORT_SYMBOL_GPL(hrtimer_start);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800461
462/**
463 * hrtimer_try_to_cancel - try to deactivate a timer
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800464 * @timer: hrtimer to stop
465 *
466 * Returns:
467 * 0 when the timer was not active
468 * 1 when the timer was active
469 * -1 when the timer is currently excuting the callback function and
Randy Dunlapfa9799e2006-06-25 05:49:15 -0700470 * cannot be stopped
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800471 */
472int hrtimer_try_to_cancel(struct hrtimer *timer)
473{
474 struct hrtimer_base *base;
475 unsigned long flags;
476 int ret = -1;
477
478 base = lock_hrtimer_base(timer, &flags);
479
480 if (base->curr_timer != timer)
481 ret = remove_hrtimer(timer, base);
482
483 unlock_hrtimer_base(timer, &flags);
484
485 return ret;
486
487}
Stephen Hemminger8d16b762006-05-30 21:26:09 -0700488EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800489
490/**
491 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800492 * @timer: the timer to be cancelled
493 *
494 * Returns:
495 * 0 when the timer was not active
496 * 1 when the timer was active
497 */
498int hrtimer_cancel(struct hrtimer *timer)
499{
500 for (;;) {
501 int ret = hrtimer_try_to_cancel(timer);
502
503 if (ret >= 0)
504 return ret;
Joe Korty5ef37b12006-04-10 22:54:13 -0700505 cpu_relax();
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800506 }
507}
Stephen Hemminger8d16b762006-05-30 21:26:09 -0700508EXPORT_SYMBOL_GPL(hrtimer_cancel);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800509
510/**
511 * hrtimer_get_remaining - get remaining time for the timer
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800512 * @timer: the timer to read
513 */
514ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
515{
516 struct hrtimer_base *base;
517 unsigned long flags;
518 ktime_t rem;
519
520 base = lock_hrtimer_base(timer, &flags);
521 rem = ktime_sub(timer->expires, timer->base->get_time());
522 unlock_hrtimer_base(timer, &flags);
523
524 return rem;
525}
Stephen Hemminger8d16b762006-05-30 21:26:09 -0700526EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800527
Tony Lindgren69239742006-03-06 15:42:45 -0800528#ifdef CONFIG_NO_IDLE_HZ
529/**
530 * hrtimer_get_next_event - get the time until next expiry event
531 *
532 * Returns the delta to the next expiry event or KTIME_MAX if no timer
533 * is pending.
534 */
535ktime_t hrtimer_get_next_event(void)
536{
537 struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
538 ktime_t delta, mindelta = { .tv64 = KTIME_MAX };
539 unsigned long flags;
540 int i;
541
542 for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) {
543 struct hrtimer *timer;
544
545 spin_lock_irqsave(&base->lock, flags);
546 if (!base->first) {
547 spin_unlock_irqrestore(&base->lock, flags);
548 continue;
549 }
550 timer = rb_entry(base->first, struct hrtimer, node);
551 delta.tv64 = timer->expires.tv64;
552 spin_unlock_irqrestore(&base->lock, flags);
553 delta = ktime_sub(delta, base->get_time());
554 if (delta.tv64 < mindelta.tv64)
555 mindelta.tv64 = delta.tv64;
556 }
557 if (mindelta.tv64 < 0)
558 mindelta.tv64 = 0;
559 return mindelta;
560}
561#endif
562
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800563/**
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800564 * hrtimer_init - initialize a timer to the given clock
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800565 * @timer: the timer to be initialized
566 * @clock_id: the clock to be used
George Anzinger7978672c2006-02-01 03:05:11 -0800567 * @mode: timer mode abs/rel
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800568 */
George Anzinger7978672c2006-02-01 03:05:11 -0800569void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
570 enum hrtimer_mode mode)
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800571{
George Anzinger7978672c2006-02-01 03:05:11 -0800572 struct hrtimer_base *bases;
573
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800574 memset(timer, 0, sizeof(struct hrtimer));
George Anzinger7978672c2006-02-01 03:05:11 -0800575
Paul Mackerrasbfe5d832006-06-25 05:47:14 -0700576 bases = __raw_get_cpu_var(hrtimer_bases);
George Anzinger7978672c2006-02-01 03:05:11 -0800577
578 if (clock_id == CLOCK_REALTIME && mode != HRTIMER_ABS)
579 clock_id = CLOCK_MONOTONIC;
580
581 timer->base = &bases[clock_id];
David Woodhouseed198cb2006-04-22 02:38:50 +0100582 rb_set_parent(&timer->node, &timer->node);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800583}
Stephen Hemminger8d16b762006-05-30 21:26:09 -0700584EXPORT_SYMBOL_GPL(hrtimer_init);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800585
586/**
587 * hrtimer_get_res - get the timer resolution for a clock
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800588 * @which_clock: which clock to query
589 * @tp: pointer to timespec variable to store the resolution
590 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800591 * Store the resolution of the clock selected by @which_clock in the
592 * variable pointed to by @tp.
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800593 */
594int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
595{
596 struct hrtimer_base *bases;
597
Paul Mackerrasbfe5d832006-06-25 05:47:14 -0700598 bases = __raw_get_cpu_var(hrtimer_bases);
Thomas Gleixnere2787632006-01-12 11:36:14 +0100599 *tp = ktime_to_timespec(bases[which_clock].resolution);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800600
601 return 0;
602}
Stephen Hemminger8d16b762006-05-30 21:26:09 -0700603EXPORT_SYMBOL_GPL(hrtimer_get_res);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800604
605/*
606 * Expire the per base hrtimer-queue:
607 */
608static inline void run_hrtimer_queue(struct hrtimer_base *base)
609{
Thomas Gleixner288867e2006-01-12 11:25:54 +0100610 struct rb_node *node;
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800611
Dimitri Sivanich3055add2006-03-31 02:31:20 -0800612 if (!base->first)
613 return;
614
Thomas Gleixner92127c72006-03-26 01:38:05 -0800615 if (base->get_softirq_time)
616 base->softirq_time = base->get_softirq_time();
617
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800618 spin_lock_irq(&base->lock);
619
Thomas Gleixner288867e2006-01-12 11:25:54 +0100620 while ((node = base->first)) {
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800621 struct hrtimer *timer;
Roman Zippel05cfb612006-03-26 01:38:12 -0800622 int (*fn)(struct hrtimer *);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800623 int restart;
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800624
Thomas Gleixner288867e2006-01-12 11:25:54 +0100625 timer = rb_entry(node, struct hrtimer, node);
Thomas Gleixner92127c72006-03-26 01:38:05 -0800626 if (base->softirq_time.tv64 <= timer->expires.tv64)
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800627 break;
628
629 fn = timer->function;
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800630 set_curr_timer(base, timer);
631 __remove_hrtimer(timer, base);
632 spin_unlock_irq(&base->lock);
633
Roman Zippel05cfb612006-03-26 01:38:12 -0800634 restart = fn(timer);
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800635
636 spin_lock_irq(&base->lock);
637
Roman Zippelb75f7a52006-03-26 01:38:09 -0800638 if (restart != HRTIMER_NORESTART) {
639 BUG_ON(hrtimer_active(timer));
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800640 enqueue_hrtimer(timer, base);
Roman Zippelb75f7a52006-03-26 01:38:09 -0800641 }
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800642 }
643 set_curr_timer(base, NULL);
644 spin_unlock_irq(&base->lock);
645}
646
647/*
648 * Called from timer softirq every jiffy, expire hrtimers:
649 */
650void hrtimer_run_queues(void)
651{
652 struct hrtimer_base *base = __get_cpu_var(hrtimer_bases);
653 int i;
654
Thomas Gleixner92127c72006-03-26 01:38:05 -0800655 hrtimer_get_softirq_time(base);
656
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800657 for (i = 0; i < MAX_HRTIMER_BASES; i++)
658 run_hrtimer_queue(&base[i]);
659}
660
661/*
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800662 * Sleep related functions:
663 */
Thomas Gleixner00362e32006-03-31 02:31:17 -0800664static int hrtimer_wakeup(struct hrtimer *timer)
665{
666 struct hrtimer_sleeper *t =
667 container_of(timer, struct hrtimer_sleeper, timer);
668 struct task_struct *task = t->task;
669
670 t->task = NULL;
671 if (task)
672 wake_up_process(task);
673
674 return HRTIMER_NORESTART;
675}
676
Ingo Molnar36c8b582006-07-03 00:25:41 -0700677void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
Thomas Gleixner00362e32006-03-31 02:31:17 -0800678{
679 sl->timer.function = hrtimer_wakeup;
680 sl->task = task;
681}
682
Thomas Gleixner669d7862006-03-31 02:31:19 -0800683static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800684{
Thomas Gleixner669d7862006-03-31 02:31:19 -0800685 hrtimer_init_sleeper(t, current);
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800686
Roman Zippel432569b2006-03-26 01:38:08 -0800687 do {
688 set_current_state(TASK_INTERRUPTIBLE);
689 hrtimer_start(&t->timer, t->timer.expires, mode);
690
691 schedule();
692
Thomas Gleixner669d7862006-03-31 02:31:19 -0800693 hrtimer_cancel(&t->timer);
694 mode = HRTIMER_ABS;
Roman Zippel432569b2006-03-26 01:38:08 -0800695
Thomas Gleixner669d7862006-03-31 02:31:19 -0800696 } while (t->task && !signal_pending(current));
697
698 return t->task == NULL;
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800699}
700
Toyo Abe1711ef32006-09-29 02:00:28 -0700701long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800702{
Thomas Gleixner669d7862006-03-31 02:31:19 -0800703 struct hrtimer_sleeper t;
Ingo Molnarea13dbc2006-01-16 10:59:41 +0100704 struct timespec __user *rmtp;
705 struct timespec tu;
Roman Zippel432569b2006-03-26 01:38:08 -0800706 ktime_t time;
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800707
708 restart->fn = do_no_restart_syscall;
709
Toyo Abe1711ef32006-09-29 02:00:28 -0700710 hrtimer_init(&t.timer, restart->arg0, HRTIMER_ABS);
711 t.timer.expires.tv64 = ((u64)restart->arg3 << 32) | (u64) restart->arg2;
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800712
Roman Zippel432569b2006-03-26 01:38:08 -0800713 if (do_nanosleep(&t, HRTIMER_ABS))
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800714 return 0;
715
Toyo Abe1711ef32006-09-29 02:00:28 -0700716 rmtp = (struct timespec __user *) restart->arg1;
Roman Zippel432569b2006-03-26 01:38:08 -0800717 if (rmtp) {
718 time = ktime_sub(t.timer.expires, t.timer.base->get_time());
719 if (time.tv64 <= 0)
720 return 0;
721 tu = ktime_to_timespec(time);
722 if (copy_to_user(rmtp, &tu, sizeof(tu)))
723 return -EFAULT;
724 }
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800725
Toyo Abe1711ef32006-09-29 02:00:28 -0700726 restart->fn = hrtimer_nanosleep_restart;
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800727
728 /* The other values in restart are already filled in */
729 return -ERESTART_RESTARTBLOCK;
730}
731
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800732long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
733 const enum hrtimer_mode mode, const clockid_t clockid)
734{
735 struct restart_block *restart;
Thomas Gleixner669d7862006-03-31 02:31:19 -0800736 struct hrtimer_sleeper t;
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800737 struct timespec tu;
738 ktime_t rem;
739
Roman Zippel432569b2006-03-26 01:38:08 -0800740 hrtimer_init(&t.timer, clockid, mode);
741 t.timer.expires = timespec_to_ktime(*rqtp);
742 if (do_nanosleep(&t, mode))
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800743 return 0;
744
George Anzinger7978672c2006-02-01 03:05:11 -0800745 /* Absolute timers do not update the rmtp value and restart: */
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800746 if (mode == HRTIMER_ABS)
747 return -ERESTARTNOHAND;
748
Roman Zippel432569b2006-03-26 01:38:08 -0800749 if (rmtp) {
750 rem = ktime_sub(t.timer.expires, t.timer.base->get_time());
751 if (rem.tv64 <= 0)
752 return 0;
753 tu = ktime_to_timespec(rem);
754 if (copy_to_user(rmtp, &tu, sizeof(tu)))
755 return -EFAULT;
756 }
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800757
758 restart = &current_thread_info()->restart_block;
Toyo Abe1711ef32006-09-29 02:00:28 -0700759 restart->fn = hrtimer_nanosleep_restart;
760 restart->arg0 = (unsigned long) t.timer.base->index;
761 restart->arg1 = (unsigned long) rmtp;
762 restart->arg2 = t.timer.expires.tv64 & 0xFFFFFFFF;
763 restart->arg3 = t.timer.expires.tv64 >> 32;
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800764
765 return -ERESTART_RESTARTBLOCK;
766}
767
Thomas Gleixner6ba1b912006-01-09 20:52:36 -0800768asmlinkage long
769sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp)
770{
771 struct timespec tu;
772
773 if (copy_from_user(&tu, rqtp, sizeof(tu)))
774 return -EFAULT;
775
776 if (!timespec_valid(&tu))
777 return -EINVAL;
778
779 return hrtimer_nanosleep(&tu, rmtp, HRTIMER_REL, CLOCK_MONOTONIC);
780}
781
Thomas Gleixner10c94ec2006-01-09 20:52:35 -0800782/*
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800783 * Functions related to boot-time initialization:
784 */
785static void __devinit init_hrtimers_cpu(int cpu)
786{
787 struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu);
788 int i;
789
Ingo Molnar54365522006-07-03 00:25:11 -0700790 for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) {
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800791 spin_lock_init(&base->lock);
Ingo Molnar54365522006-07-03 00:25:11 -0700792 lockdep_set_class(&base->lock, &base->lock_key);
793 }
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800794}
795
796#ifdef CONFIG_HOTPLUG_CPU
797
798static void migrate_hrtimer_list(struct hrtimer_base *old_base,
799 struct hrtimer_base *new_base)
800{
801 struct hrtimer *timer;
802 struct rb_node *node;
803
804 while ((node = rb_first(&old_base->active))) {
805 timer = rb_entry(node, struct hrtimer, node);
806 __remove_hrtimer(timer, old_base);
807 timer->base = new_base;
808 enqueue_hrtimer(timer, new_base);
809 }
810}
811
812static void migrate_hrtimers(int cpu)
813{
814 struct hrtimer_base *old_base, *new_base;
815 int i;
816
817 BUG_ON(cpu_online(cpu));
818 old_base = per_cpu(hrtimer_bases, cpu);
819 new_base = get_cpu_var(hrtimer_bases);
820
821 local_irq_disable();
822
823 for (i = 0; i < MAX_HRTIMER_BASES; i++) {
824
825 spin_lock(&new_base->lock);
826 spin_lock(&old_base->lock);
827
828 BUG_ON(old_base->curr_timer);
829
830 migrate_hrtimer_list(old_base, new_base);
831
832 spin_unlock(&old_base->lock);
833 spin_unlock(&new_base->lock);
834 old_base++;
835 new_base++;
836 }
837
838 local_irq_enable();
839 put_cpu_var(hrtimer_bases);
840}
841#endif /* CONFIG_HOTPLUG_CPU */
842
Chandra Seetharaman8c78f302006-07-30 03:03:35 -0700843static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800844 unsigned long action, void *hcpu)
845{
846 long cpu = (long)hcpu;
847
848 switch (action) {
849
850 case CPU_UP_PREPARE:
851 init_hrtimers_cpu(cpu);
852 break;
853
854#ifdef CONFIG_HOTPLUG_CPU
855 case CPU_DEAD:
856 migrate_hrtimers(cpu);
857 break;
858#endif
859
860 default:
861 break;
862 }
863
864 return NOTIFY_OK;
865}
866
Chandra Seetharaman8c78f302006-07-30 03:03:35 -0700867static struct notifier_block __cpuinitdata hrtimers_nb = {
Thomas Gleixnerc0a31322006-01-09 20:52:32 -0800868 .notifier_call = hrtimer_cpu_notify,
869};
870
871void __init hrtimers_init(void)
872{
873 hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
874 (void *)(long)smp_processor_id());
875 register_cpu_notifier(&hrtimers_nb);
876}
877