xref: /linux/kernel/time/hrtimer.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
4  *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
5  *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
6  *
7  *  High-resolution kernel timers
8  *
9  *  In contrast to the low-resolution timeout API, aka timer wheel,
10  *  hrtimers provide finer resolution and accuracy depending on system
11  *  configuration and capabilities.
12  *
13  *  Started by: Thomas Gleixner and Ingo Molnar
14  *
15  *  Credits:
16  *	Based on the original timer wheel code
17  *
18  *	Help, testing, suggestions, bugfixes, improvements were
19  *	provided by:
20  *
21  *	George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
22  *	et. al.
23  */
24 
25 #include <linux/cpu.h>
26 #include <linux/export.h>
27 #include <linux/percpu.h>
28 #include <linux/hrtimer.h>
29 #include <linux/notifier.h>
30 #include <linux/syscalls.h>
31 #include <linux/interrupt.h>
32 #include <linux/tick.h>
33 #include <linux/err.h>
34 #include <linux/debugobjects.h>
35 #include <linux/sched/signal.h>
36 #include <linux/sched/sysctl.h>
37 #include <linux/sched/rt.h>
38 #include <linux/sched/deadline.h>
39 #include <linux/sched/nohz.h>
40 #include <linux/sched/debug.h>
41 #include <linux/sched/isolation.h>
42 #include <linux/timer.h>
43 #include <linux/freezer.h>
44 #include <linux/compat.h>
45 
46 #include <linux/uaccess.h>
47 
48 #include <trace/events/timer.h>
49 
50 #include "tick-internal.h"
51 
52 /*
53  * Masks for selecting the soft and hard context timers from
54  * cpu_base->active
55  */
56 #define MASK_SHIFT		(HRTIMER_BASE_MONOTONIC_SOFT)
57 #define HRTIMER_ACTIVE_HARD	((1U << MASK_SHIFT) - 1)
58 #define HRTIMER_ACTIVE_SOFT	(HRTIMER_ACTIVE_HARD << MASK_SHIFT)
59 #define HRTIMER_ACTIVE_ALL	(HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
60 
61 static void retrigger_next_event(void *arg);
62 
63 /*
64  * The timer bases:
65  *
66  * There are more clockids than hrtimer bases. Thus, we index
67  * into the timer bases by the hrtimer_base_type enum. When trying
68  * to reach a base using a clockid, hrtimer_clockid_to_base()
69  * is used to convert from clockid to the proper hrtimer_base_type.
70  */
71 DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
72 {
73 	.lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
74 	.clock_base =
75 	{
76 		{
77 			.index = HRTIMER_BASE_MONOTONIC,
78 			.clockid = CLOCK_MONOTONIC,
79 			.get_time = &ktime_get,
80 		},
81 		{
82 			.index = HRTIMER_BASE_REALTIME,
83 			.clockid = CLOCK_REALTIME,
84 			.get_time = &ktime_get_real,
85 		},
86 		{
87 			.index = HRTIMER_BASE_BOOTTIME,
88 			.clockid = CLOCK_BOOTTIME,
89 			.get_time = &ktime_get_boottime,
90 		},
91 		{
92 			.index = HRTIMER_BASE_TAI,
93 			.clockid = CLOCK_TAI,
94 			.get_time = &ktime_get_clocktai,
95 		},
96 		{
97 			.index = HRTIMER_BASE_MONOTONIC_SOFT,
98 			.clockid = CLOCK_MONOTONIC,
99 			.get_time = &ktime_get,
100 		},
101 		{
102 			.index = HRTIMER_BASE_REALTIME_SOFT,
103 			.clockid = CLOCK_REALTIME,
104 			.get_time = &ktime_get_real,
105 		},
106 		{
107 			.index = HRTIMER_BASE_BOOTTIME_SOFT,
108 			.clockid = CLOCK_BOOTTIME,
109 			.get_time = &ktime_get_boottime,
110 		},
111 		{
112 			.index = HRTIMER_BASE_TAI_SOFT,
113 			.clockid = CLOCK_TAI,
114 			.get_time = &ktime_get_clocktai,
115 		},
116 	},
117 	.csd = CSD_INIT(retrigger_next_event, NULL)
118 };
119 
120 static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
121 	/* Make sure we catch unsupported clockids */
122 	[0 ... MAX_CLOCKS - 1]	= HRTIMER_MAX_CLOCK_BASES,
123 
124 	[CLOCK_REALTIME]	= HRTIMER_BASE_REALTIME,
125 	[CLOCK_MONOTONIC]	= HRTIMER_BASE_MONOTONIC,
126 	[CLOCK_BOOTTIME]	= HRTIMER_BASE_BOOTTIME,
127 	[CLOCK_TAI]		= HRTIMER_BASE_TAI,
128 };
129 
hrtimer_base_is_online(struct hrtimer_cpu_base * base)130 static inline bool hrtimer_base_is_online(struct hrtimer_cpu_base *base)
131 {
132 	if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
133 		return true;
134 	else
135 		return likely(base->online);
136 }
137 
138 /*
139  * Functions and macros which are different for UP/SMP systems are kept in a
140  * single place
141  */
142 #ifdef CONFIG_SMP
143 
144 /*
145  * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
146  * such that hrtimer_callback_running() can unconditionally dereference
147  * timer->base->cpu_base
148  */
149 static struct hrtimer_cpu_base migration_cpu_base = {
150 	.clock_base = { {
151 		.cpu_base = &migration_cpu_base,
152 		.seq      = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
153 						     &migration_cpu_base.lock),
154 	}, },
155 };
156 
157 #define migration_base	migration_cpu_base.clock_base[0]
158 
159 /*
160  * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
161  * means that all timers which are tied to this base via timer->base are
162  * locked, and the base itself is locked too.
163  *
164  * So __run_timers/migrate_timers can safely modify all timers which could
165  * be found on the lists/queues.
166  *
167  * When the timer's base is locked, and the timer removed from list, it is
168  * possible to set timer->base = &migration_base and drop the lock: the timer
169  * remains locked.
170  */
171 static
lock_hrtimer_base(const struct hrtimer * timer,unsigned long * flags)172 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
173 					     unsigned long *flags)
174 	__acquires(&timer->base->lock)
175 {
176 	struct hrtimer_clock_base *base;
177 
178 	for (;;) {
179 		base = READ_ONCE(timer->base);
180 		if (likely(base != &migration_base)) {
181 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
182 			if (likely(base == timer->base))
183 				return base;
184 			/* The timer has migrated to another CPU: */
185 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
186 		}
187 		cpu_relax();
188 	}
189 }
190 
191 /*
192  * Check if the elected target is suitable considering its next
193  * event and the hotplug state of the current CPU.
194  *
195  * If the elected target is remote and its next event is after the timer
196  * to queue, then a remote reprogram is necessary. However there is no
197  * guarantee the IPI handling the operation would arrive in time to meet
198  * the high resolution deadline. In this case the local CPU becomes a
199  * preferred target, unless it is offline.
200  *
201  * High and low resolution modes are handled the same way for simplicity.
202  *
203  * Called with cpu_base->lock of target cpu held.
204  */
hrtimer_suitable_target(struct hrtimer * timer,struct hrtimer_clock_base * new_base,struct hrtimer_cpu_base * new_cpu_base,struct hrtimer_cpu_base * this_cpu_base)205 static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base,
206 				    struct hrtimer_cpu_base *new_cpu_base,
207 				    struct hrtimer_cpu_base *this_cpu_base)
208 {
209 	ktime_t expires;
210 
211 	/*
212 	 * The local CPU clockevent can be reprogrammed. Also get_target_base()
213 	 * guarantees it is online.
214 	 */
215 	if (new_cpu_base == this_cpu_base)
216 		return true;
217 
218 	/*
219 	 * The offline local CPU can't be the default target if the
220 	 * next remote target event is after this timer. Keep the
221 	 * elected new base. An IPI will we issued to reprogram
222 	 * it as a last resort.
223 	 */
224 	if (!hrtimer_base_is_online(this_cpu_base))
225 		return true;
226 
227 	expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
228 
229 	return expires >= new_base->cpu_base->expires_next;
230 }
231 
get_target_base(struct hrtimer_cpu_base * base,int pinned)232 static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, int pinned)
233 {
234 	if (!hrtimer_base_is_online(base)) {
235 		int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER));
236 
237 		return &per_cpu(hrtimer_bases, cpu);
238 	}
239 
240 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
241 	if (static_branch_likely(&timers_migration_enabled) && !pinned)
242 		return &per_cpu(hrtimer_bases, get_nohz_timer_target());
243 #endif
244 	return base;
245 }
246 
247 /*
248  * We switch the timer base to a power-optimized selected CPU target,
249  * if:
250  *	- NO_HZ_COMMON is enabled
251  *	- timer migration is enabled
252  *	- the timer callback is not running
253  *	- the timer is not the first expiring timer on the new target
254  *
255  * If one of the above requirements is not fulfilled we move the timer
256  * to the current CPU or leave it on the previously assigned CPU if
257  * the timer callback is currently running.
258  */
259 static inline struct hrtimer_clock_base *
switch_hrtimer_base(struct hrtimer * timer,struct hrtimer_clock_base * base,int pinned)260 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
261 		    int pinned)
262 {
263 	struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
264 	struct hrtimer_clock_base *new_base;
265 	int basenum = base->index;
266 
267 	this_cpu_base = this_cpu_ptr(&hrtimer_bases);
268 	new_cpu_base = get_target_base(this_cpu_base, pinned);
269 again:
270 	new_base = &new_cpu_base->clock_base[basenum];
271 
272 	if (base != new_base) {
273 		/*
274 		 * We are trying to move timer to new_base.
275 		 * However we can't change timer's base while it is running,
276 		 * so we keep it on the same CPU. No hassle vs. reprogramming
277 		 * the event source in the high resolution case. The softirq
278 		 * code will take care of this when the timer function has
279 		 * completed. There is no conflict as we hold the lock until
280 		 * the timer is enqueued.
281 		 */
282 		if (unlikely(hrtimer_callback_running(timer)))
283 			return base;
284 
285 		/* See the comment in lock_hrtimer_base() */
286 		WRITE_ONCE(timer->base, &migration_base);
287 		raw_spin_unlock(&base->cpu_base->lock);
288 		raw_spin_lock(&new_base->cpu_base->lock);
289 
290 		if (!hrtimer_suitable_target(timer, new_base, new_cpu_base,
291 					     this_cpu_base)) {
292 			raw_spin_unlock(&new_base->cpu_base->lock);
293 			raw_spin_lock(&base->cpu_base->lock);
294 			new_cpu_base = this_cpu_base;
295 			WRITE_ONCE(timer->base, base);
296 			goto again;
297 		}
298 		WRITE_ONCE(timer->base, new_base);
299 	} else {
300 		if (!hrtimer_suitable_target(timer, new_base,  new_cpu_base, this_cpu_base)) {
301 			new_cpu_base = this_cpu_base;
302 			goto again;
303 		}
304 	}
305 	return new_base;
306 }
307 
308 #else /* CONFIG_SMP */
309 
310 static inline struct hrtimer_clock_base *
lock_hrtimer_base(const struct hrtimer * timer,unsigned long * flags)311 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
312 	__acquires(&timer->base->cpu_base->lock)
313 {
314 	struct hrtimer_clock_base *base = timer->base;
315 
316 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
317 
318 	return base;
319 }
320 
321 # define switch_hrtimer_base(t, b, p)	(b)
322 
323 #endif	/* !CONFIG_SMP */
324 
325 /*
326  * Functions for the union type storage format of ktime_t which are
327  * too large for inlining:
328  */
329 #if BITS_PER_LONG < 64
330 /*
331  * Divide a ktime value by a nanosecond value
332  */
__ktime_divns(const ktime_t kt,s64 div)333 s64 __ktime_divns(const ktime_t kt, s64 div)
334 {
335 	int sft = 0;
336 	s64 dclc;
337 	u64 tmp;
338 
339 	dclc = ktime_to_ns(kt);
340 	tmp = dclc < 0 ? -dclc : dclc;
341 
342 	/* Make sure the divisor is less than 2^32: */
343 	while (div >> 32) {
344 		sft++;
345 		div >>= 1;
346 	}
347 	tmp >>= sft;
348 	do_div(tmp, (u32) div);
349 	return dclc < 0 ? -tmp : tmp;
350 }
351 EXPORT_SYMBOL_GPL(__ktime_divns);
352 #endif /* BITS_PER_LONG >= 64 */
353 
354 /*
355  * Add two ktime values and do a safety check for overflow:
356  */
ktime_add_safe(const ktime_t lhs,const ktime_t rhs)357 ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
358 {
359 	ktime_t res = ktime_add_unsafe(lhs, rhs);
360 
361 	/*
362 	 * We use KTIME_SEC_MAX here, the maximum timeout which we can
363 	 * return to user space in a timespec:
364 	 */
365 	if (res < 0 || res < lhs || res < rhs)
366 		res = ktime_set(KTIME_SEC_MAX, 0);
367 
368 	return res;
369 }
370 
371 EXPORT_SYMBOL_GPL(ktime_add_safe);
372 
373 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
374 
375 static const struct debug_obj_descr hrtimer_debug_descr;
376 
hrtimer_debug_hint(void * addr)377 static void *hrtimer_debug_hint(void *addr)
378 {
379 	return ((struct hrtimer *) addr)->function;
380 }
381 
382 /*
383  * fixup_init is called when:
384  * - an active object is initialized
385  */
hrtimer_fixup_init(void * addr,enum debug_obj_state state)386 static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
387 {
388 	struct hrtimer *timer = addr;
389 
390 	switch (state) {
391 	case ODEBUG_STATE_ACTIVE:
392 		hrtimer_cancel(timer);
393 		debug_object_init(timer, &hrtimer_debug_descr);
394 		return true;
395 	default:
396 		return false;
397 	}
398 }
399 
400 /*
401  * fixup_activate is called when:
402  * - an active object is activated
403  * - an unknown non-static object is activated
404  */
hrtimer_fixup_activate(void * addr,enum debug_obj_state state)405 static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
406 {
407 	switch (state) {
408 	case ODEBUG_STATE_ACTIVE:
409 		WARN_ON(1);
410 		fallthrough;
411 	default:
412 		return false;
413 	}
414 }
415 
416 /*
417  * fixup_free is called when:
418  * - an active object is freed
419  */
hrtimer_fixup_free(void * addr,enum debug_obj_state state)420 static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
421 {
422 	struct hrtimer *timer = addr;
423 
424 	switch (state) {
425 	case ODEBUG_STATE_ACTIVE:
426 		hrtimer_cancel(timer);
427 		debug_object_free(timer, &hrtimer_debug_descr);
428 		return true;
429 	default:
430 		return false;
431 	}
432 }
433 
434 static const struct debug_obj_descr hrtimer_debug_descr = {
435 	.name		= "hrtimer",
436 	.debug_hint	= hrtimer_debug_hint,
437 	.fixup_init	= hrtimer_fixup_init,
438 	.fixup_activate	= hrtimer_fixup_activate,
439 	.fixup_free	= hrtimer_fixup_free,
440 };
441 
debug_hrtimer_init(struct hrtimer * timer)442 static inline void debug_hrtimer_init(struct hrtimer *timer)
443 {
444 	debug_object_init(timer, &hrtimer_debug_descr);
445 }
446 
debug_hrtimer_init_on_stack(struct hrtimer * timer)447 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer)
448 {
449 	debug_object_init_on_stack(timer, &hrtimer_debug_descr);
450 }
451 
debug_hrtimer_activate(struct hrtimer * timer,enum hrtimer_mode mode)452 static inline void debug_hrtimer_activate(struct hrtimer *timer,
453 					  enum hrtimer_mode mode)
454 {
455 	debug_object_activate(timer, &hrtimer_debug_descr);
456 }
457 
debug_hrtimer_deactivate(struct hrtimer * timer)458 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
459 {
460 	debug_object_deactivate(timer, &hrtimer_debug_descr);
461 }
462 
destroy_hrtimer_on_stack(struct hrtimer * timer)463 void destroy_hrtimer_on_stack(struct hrtimer *timer)
464 {
465 	debug_object_free(timer, &hrtimer_debug_descr);
466 }
467 EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
468 
469 #else
470 
debug_hrtimer_init(struct hrtimer * timer)471 static inline void debug_hrtimer_init(struct hrtimer *timer) { }
debug_hrtimer_init_on_stack(struct hrtimer * timer)472 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }
debug_hrtimer_activate(struct hrtimer * timer,enum hrtimer_mode mode)473 static inline void debug_hrtimer_activate(struct hrtimer *timer,
474 					  enum hrtimer_mode mode) { }
debug_hrtimer_deactivate(struct hrtimer * timer)475 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
476 #endif
477 
478 static inline void
debug_init(struct hrtimer * timer,clockid_t clockid,enum hrtimer_mode mode)479 debug_init(struct hrtimer *timer, clockid_t clockid,
480 	   enum hrtimer_mode mode)
481 {
482 	debug_hrtimer_init(timer);
483 	trace_hrtimer_init(timer, clockid, mode);
484 }
485 
debug_init_on_stack(struct hrtimer * timer,clockid_t clockid,enum hrtimer_mode mode)486 static inline void debug_init_on_stack(struct hrtimer *timer, clockid_t clockid,
487 				       enum hrtimer_mode mode)
488 {
489 	debug_hrtimer_init_on_stack(timer);
490 	trace_hrtimer_init(timer, clockid, mode);
491 }
492 
debug_activate(struct hrtimer * timer,enum hrtimer_mode mode)493 static inline void debug_activate(struct hrtimer *timer,
494 				  enum hrtimer_mode mode)
495 {
496 	debug_hrtimer_activate(timer, mode);
497 	trace_hrtimer_start(timer, mode);
498 }
499 
debug_deactivate(struct hrtimer * timer)500 static inline void debug_deactivate(struct hrtimer *timer)
501 {
502 	debug_hrtimer_deactivate(timer);
503 	trace_hrtimer_cancel(timer);
504 }
505 
506 static struct hrtimer_clock_base *
__next_base(struct hrtimer_cpu_base * cpu_base,unsigned int * active)507 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
508 {
509 	unsigned int idx;
510 
511 	if (!*active)
512 		return NULL;
513 
514 	idx = __ffs(*active);
515 	*active &= ~(1U << idx);
516 
517 	return &cpu_base->clock_base[idx];
518 }
519 
520 #define for_each_active_base(base, cpu_base, active)	\
521 	while ((base = __next_base((cpu_base), &(active))))
522 
__hrtimer_next_event_base(struct hrtimer_cpu_base * cpu_base,const struct hrtimer * exclude,unsigned int active,ktime_t expires_next)523 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
524 					 const struct hrtimer *exclude,
525 					 unsigned int active,
526 					 ktime_t expires_next)
527 {
528 	struct hrtimer_clock_base *base;
529 	ktime_t expires;
530 
531 	for_each_active_base(base, cpu_base, active) {
532 		struct timerqueue_node *next;
533 		struct hrtimer *timer;
534 
535 		next = timerqueue_getnext(&base->active);
536 		timer = container_of(next, struct hrtimer, node);
537 		if (timer == exclude) {
538 			/* Get to the next timer in the queue. */
539 			next = timerqueue_iterate_next(next);
540 			if (!next)
541 				continue;
542 
543 			timer = container_of(next, struct hrtimer, node);
544 		}
545 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
546 		if (expires < expires_next) {
547 			expires_next = expires;
548 
549 			/* Skip cpu_base update if a timer is being excluded. */
550 			if (exclude)
551 				continue;
552 
553 			if (timer->is_soft)
554 				cpu_base->softirq_next_timer = timer;
555 			else
556 				cpu_base->next_timer = timer;
557 		}
558 	}
559 	/*
560 	 * clock_was_set() might have changed base->offset of any of
561 	 * the clock bases so the result might be negative. Fix it up
562 	 * to prevent a false positive in clockevents_program_event().
563 	 */
564 	if (expires_next < 0)
565 		expires_next = 0;
566 	return expires_next;
567 }
568 
569 /*
570  * Recomputes cpu_base::*next_timer and returns the earliest expires_next
571  * but does not set cpu_base::*expires_next, that is done by
572  * hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
573  * cpu_base::*expires_next right away, reprogramming logic would no longer
574  * work.
575  *
576  * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
577  * those timers will get run whenever the softirq gets handled, at the end of
578  * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
579  *
580  * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
581  * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
582  * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
583  *
584  * @active_mask must be one of:
585  *  - HRTIMER_ACTIVE_ALL,
586  *  - HRTIMER_ACTIVE_SOFT, or
587  *  - HRTIMER_ACTIVE_HARD.
588  */
589 static ktime_t
__hrtimer_get_next_event(struct hrtimer_cpu_base * cpu_base,unsigned int active_mask)590 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
591 {
592 	unsigned int active;
593 	struct hrtimer *next_timer = NULL;
594 	ktime_t expires_next = KTIME_MAX;
595 
596 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
597 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
598 		cpu_base->softirq_next_timer = NULL;
599 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,
600 							 active, KTIME_MAX);
601 
602 		next_timer = cpu_base->softirq_next_timer;
603 	}
604 
605 	if (active_mask & HRTIMER_ACTIVE_HARD) {
606 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
607 		cpu_base->next_timer = next_timer;
608 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,
609 							 expires_next);
610 	}
611 
612 	return expires_next;
613 }
614 
hrtimer_update_next_event(struct hrtimer_cpu_base * cpu_base)615 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
616 {
617 	ktime_t expires_next, soft = KTIME_MAX;
618 
619 	/*
620 	 * If the soft interrupt has already been activated, ignore the
621 	 * soft bases. They will be handled in the already raised soft
622 	 * interrupt.
623 	 */
624 	if (!cpu_base->softirq_activated) {
625 		soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
626 		/*
627 		 * Update the soft expiry time. clock_settime() might have
628 		 * affected it.
629 		 */
630 		cpu_base->softirq_expires_next = soft;
631 	}
632 
633 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
634 	/*
635 	 * If a softirq timer is expiring first, update cpu_base->next_timer
636 	 * and program the hardware with the soft expiry time.
637 	 */
638 	if (expires_next > soft) {
639 		cpu_base->next_timer = cpu_base->softirq_next_timer;
640 		expires_next = soft;
641 	}
642 
643 	return expires_next;
644 }
645 
hrtimer_update_base(struct hrtimer_cpu_base * base)646 static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
647 {
648 	ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
649 	ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
650 	ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
651 
652 	ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
653 					    offs_real, offs_boot, offs_tai);
654 
655 	base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
656 	base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
657 	base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
658 
659 	return now;
660 }
661 
662 /*
663  * Is the high resolution mode active ?
664  */
hrtimer_hres_active(struct hrtimer_cpu_base * cpu_base)665 static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
666 {
667 	return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
668 		cpu_base->hres_active : 0;
669 }
670 
__hrtimer_reprogram(struct hrtimer_cpu_base * cpu_base,struct hrtimer * next_timer,ktime_t expires_next)671 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
672 				struct hrtimer *next_timer,
673 				ktime_t expires_next)
674 {
675 	cpu_base->expires_next = expires_next;
676 
677 	/*
678 	 * If hres is not active, hardware does not have to be
679 	 * reprogrammed yet.
680 	 *
681 	 * If a hang was detected in the last timer interrupt then we
682 	 * leave the hang delay active in the hardware. We want the
683 	 * system to make progress. That also prevents the following
684 	 * scenario:
685 	 * T1 expires 50ms from now
686 	 * T2 expires 5s from now
687 	 *
688 	 * T1 is removed, so this code is called and would reprogram
689 	 * the hardware to 5s from now. Any hrtimer_start after that
690 	 * will not reprogram the hardware due to hang_detected being
691 	 * set. So we'd effectively block all timers until the T2 event
692 	 * fires.
693 	 */
694 	if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
695 		return;
696 
697 	tick_program_event(expires_next, 1);
698 }
699 
700 /*
701  * Reprogram the event source with checking both queues for the
702  * next event
703  * Called with interrupts disabled and base->lock held
704  */
705 static void
hrtimer_force_reprogram(struct hrtimer_cpu_base * cpu_base,int skip_equal)706 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
707 {
708 	ktime_t expires_next;
709 
710 	expires_next = hrtimer_update_next_event(cpu_base);
711 
712 	if (skip_equal && expires_next == cpu_base->expires_next)
713 		return;
714 
715 	__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
716 }
717 
718 /* High resolution timer related functions */
719 #ifdef CONFIG_HIGH_RES_TIMERS
720 
721 /*
722  * High resolution timer enabled ?
723  */
724 static bool hrtimer_hres_enabled __read_mostly  = true;
725 unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
726 EXPORT_SYMBOL_GPL(hrtimer_resolution);
727 
728 /*
729  * Enable / Disable high resolution mode
730  */
setup_hrtimer_hres(char * str)731 static int __init setup_hrtimer_hres(char *str)
732 {
733 	return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
734 }
735 
736 __setup("highres=", setup_hrtimer_hres);
737 
738 /*
739  * hrtimer_high_res_enabled - query, if the highres mode is enabled
740  */
hrtimer_is_hres_enabled(void)741 static inline int hrtimer_is_hres_enabled(void)
742 {
743 	return hrtimer_hres_enabled;
744 }
745 
746 /*
747  * Switch to high resolution mode
748  */
hrtimer_switch_to_hres(void)749 static void hrtimer_switch_to_hres(void)
750 {
751 	struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
752 
753 	if (tick_init_highres()) {
754 		pr_warn("Could not switch to high resolution mode on CPU %u\n",
755 			base->cpu);
756 		return;
757 	}
758 	base->hres_active = 1;
759 	hrtimer_resolution = HIGH_RES_NSEC;
760 
761 	tick_setup_sched_timer(true);
762 	/* "Retrigger" the interrupt to get things going */
763 	retrigger_next_event(NULL);
764 }
765 
766 #else
767 
hrtimer_is_hres_enabled(void)768 static inline int hrtimer_is_hres_enabled(void) { return 0; }
hrtimer_switch_to_hres(void)769 static inline void hrtimer_switch_to_hres(void) { }
770 
771 #endif /* CONFIG_HIGH_RES_TIMERS */
772 /*
773  * Retrigger next event is called after clock was set with interrupts
774  * disabled through an SMP function call or directly from low level
775  * resume code.
776  *
777  * This is only invoked when:
778  *	- CONFIG_HIGH_RES_TIMERS is enabled.
779  *	- CONFIG_NOHZ_COMMON is enabled
780  *
781  * For the other cases this function is empty and because the call sites
782  * are optimized out it vanishes as well, i.e. no need for lots of
783  * #ifdeffery.
784  */
retrigger_next_event(void * arg)785 static void retrigger_next_event(void *arg)
786 {
787 	struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
788 
789 	/*
790 	 * When high resolution mode or nohz is active, then the offsets of
791 	 * CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the
792 	 * next tick will take care of that.
793 	 *
794 	 * If high resolution mode is active then the next expiring timer
795 	 * must be reevaluated and the clock event device reprogrammed if
796 	 * necessary.
797 	 *
798 	 * In the NOHZ case the update of the offset and the reevaluation
799 	 * of the next expiring timer is enough. The return from the SMP
800 	 * function call will take care of the reprogramming in case the
801 	 * CPU was in a NOHZ idle sleep.
802 	 */
803 	if (!hrtimer_hres_active(base) && !tick_nohz_active)
804 		return;
805 
806 	raw_spin_lock(&base->lock);
807 	hrtimer_update_base(base);
808 	if (hrtimer_hres_active(base))
809 		hrtimer_force_reprogram(base, 0);
810 	else
811 		hrtimer_update_next_event(base);
812 	raw_spin_unlock(&base->lock);
813 }
814 
815 /*
816  * When a timer is enqueued and expires earlier than the already enqueued
817  * timers, we have to check, whether it expires earlier than the timer for
818  * which the clock event device was armed.
819  *
820  * Called with interrupts disabled and base->cpu_base.lock held
821  */
hrtimer_reprogram(struct hrtimer * timer,bool reprogram)822 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
823 {
824 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
825 	struct hrtimer_clock_base *base = timer->base;
826 	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
827 
828 	WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
829 
830 	/*
831 	 * CLOCK_REALTIME timer might be requested with an absolute
832 	 * expiry time which is less than base->offset. Set it to 0.
833 	 */
834 	if (expires < 0)
835 		expires = 0;
836 
837 	if (timer->is_soft) {
838 		/*
839 		 * soft hrtimer could be started on a remote CPU. In this
840 		 * case softirq_expires_next needs to be updated on the
841 		 * remote CPU. The soft hrtimer will not expire before the
842 		 * first hard hrtimer on the remote CPU -
843 		 * hrtimer_check_target() prevents this case.
844 		 */
845 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
846 
847 		if (timer_cpu_base->softirq_activated)
848 			return;
849 
850 		if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
851 			return;
852 
853 		timer_cpu_base->softirq_next_timer = timer;
854 		timer_cpu_base->softirq_expires_next = expires;
855 
856 		if (!ktime_before(expires, timer_cpu_base->expires_next) ||
857 		    !reprogram)
858 			return;
859 	}
860 
861 	/*
862 	 * If the timer is not on the current cpu, we cannot reprogram
863 	 * the other cpus clock event device.
864 	 */
865 	if (base->cpu_base != cpu_base)
866 		return;
867 
868 	if (expires >= cpu_base->expires_next)
869 		return;
870 
871 	/*
872 	 * If the hrtimer interrupt is running, then it will reevaluate the
873 	 * clock bases and reprogram the clock event device.
874 	 */
875 	if (cpu_base->in_hrtirq)
876 		return;
877 
878 	cpu_base->next_timer = timer;
879 
880 	__hrtimer_reprogram(cpu_base, timer, expires);
881 }
882 
update_needs_ipi(struct hrtimer_cpu_base * cpu_base,unsigned int active)883 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,
884 			     unsigned int active)
885 {
886 	struct hrtimer_clock_base *base;
887 	unsigned int seq;
888 	ktime_t expires;
889 
890 	/*
891 	 * Update the base offsets unconditionally so the following
892 	 * checks whether the SMP function call is required works.
893 	 *
894 	 * The update is safe even when the remote CPU is in the hrtimer
895 	 * interrupt or the hrtimer soft interrupt and expiring affected
896 	 * bases. Either it will see the update before handling a base or
897 	 * it will see it when it finishes the processing and reevaluates
898 	 * the next expiring timer.
899 	 */
900 	seq = cpu_base->clock_was_set_seq;
901 	hrtimer_update_base(cpu_base);
902 
903 	/*
904 	 * If the sequence did not change over the update then the
905 	 * remote CPU already handled it.
906 	 */
907 	if (seq == cpu_base->clock_was_set_seq)
908 		return false;
909 
910 	/*
911 	 * If the remote CPU is currently handling an hrtimer interrupt, it
912 	 * will reevaluate the first expiring timer of all clock bases
913 	 * before reprogramming. Nothing to do here.
914 	 */
915 	if (cpu_base->in_hrtirq)
916 		return false;
917 
918 	/*
919 	 * Walk the affected clock bases and check whether the first expiring
920 	 * timer in a clock base is moving ahead of the first expiring timer of
921 	 * @cpu_base. If so, the IPI must be invoked because per CPU clock
922 	 * event devices cannot be remotely reprogrammed.
923 	 */
924 	active &= cpu_base->active_bases;
925 
926 	for_each_active_base(base, cpu_base, active) {
927 		struct timerqueue_node *next;
928 
929 		next = timerqueue_getnext(&base->active);
930 		expires = ktime_sub(next->expires, base->offset);
931 		if (expires < cpu_base->expires_next)
932 			return true;
933 
934 		/* Extra check for softirq clock bases */
935 		if (base->clockid < HRTIMER_BASE_MONOTONIC_SOFT)
936 			continue;
937 		if (cpu_base->softirq_activated)
938 			continue;
939 		if (expires < cpu_base->softirq_expires_next)
940 			return true;
941 	}
942 	return false;
943 }
944 
945 /*
946  * Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and
947  * CLOCK_BOOTTIME (for late sleep time injection).
948  *
949  * This requires to update the offsets for these clocks
950  * vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this
951  * also requires to eventually reprogram the per CPU clock event devices
952  * when the change moves an affected timer ahead of the first expiring
953  * timer on that CPU. Obviously remote per CPU clock event devices cannot
954  * be reprogrammed. The other reason why an IPI has to be sent is when the
955  * system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets
956  * in the tick, which obviously might be stopped, so this has to bring out
957  * the remote CPU which might sleep in idle to get this sorted.
958  */
clock_was_set(unsigned int bases)959 void clock_was_set(unsigned int bases)
960 {
961 	struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);
962 	cpumask_var_t mask;
963 	int cpu;
964 
965 	if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active)
966 		goto out_timerfd;
967 
968 	if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
969 		on_each_cpu(retrigger_next_event, NULL, 1);
970 		goto out_timerfd;
971 	}
972 
973 	/* Avoid interrupting CPUs if possible */
974 	cpus_read_lock();
975 	for_each_online_cpu(cpu) {
976 		unsigned long flags;
977 
978 		cpu_base = &per_cpu(hrtimer_bases, cpu);
979 		raw_spin_lock_irqsave(&cpu_base->lock, flags);
980 
981 		if (update_needs_ipi(cpu_base, bases))
982 			cpumask_set_cpu(cpu, mask);
983 
984 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
985 	}
986 
987 	preempt_disable();
988 	smp_call_function_many(mask, retrigger_next_event, NULL, 1);
989 	preempt_enable();
990 	cpus_read_unlock();
991 	free_cpumask_var(mask);
992 
993 out_timerfd:
994 	timerfd_clock_was_set();
995 }
996 
clock_was_set_work(struct work_struct * work)997 static void clock_was_set_work(struct work_struct *work)
998 {
999 	clock_was_set(CLOCK_SET_WALL);
1000 }
1001 
1002 static DECLARE_WORK(hrtimer_work, clock_was_set_work);
1003 
1004 /*
1005  * Called from timekeeping code to reprogram the hrtimer interrupt device
1006  * on all cpus and to notify timerfd.
1007  */
clock_was_set_delayed(void)1008 void clock_was_set_delayed(void)
1009 {
1010 	schedule_work(&hrtimer_work);
1011 }
1012 
1013 /*
1014  * Called during resume either directly from via timekeeping_resume()
1015  * or in the case of s2idle from tick_unfreeze() to ensure that the
1016  * hrtimers are up to date.
1017  */
hrtimers_resume_local(void)1018 void hrtimers_resume_local(void)
1019 {
1020 	lockdep_assert_irqs_disabled();
1021 	/* Retrigger on the local CPU */
1022 	retrigger_next_event(NULL);
1023 }
1024 
1025 /*
1026  * Counterpart to lock_hrtimer_base above:
1027  */
1028 static inline
unlock_hrtimer_base(const struct hrtimer * timer,unsigned long * flags)1029 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
1030 	__releases(&timer->base->cpu_base->lock)
1031 {
1032 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
1033 }
1034 
1035 /**
1036  * hrtimer_forward() - forward the timer expiry
1037  * @timer:	hrtimer to forward
1038  * @now:	forward past this time
1039  * @interval:	the interval to forward
1040  *
1041  * Forward the timer expiry so it will expire in the future.
1042  *
1043  * .. note::
1044  *  This only updates the timer expiry value and does not requeue the timer.
1045  *
1046  * There is also a variant of the function hrtimer_forward_now().
1047  *
1048  * Context: Can be safely called from the callback function of @timer. If called
1049  *          from other contexts @timer must neither be enqueued nor running the
1050  *          callback and the caller needs to take care of serialization.
1051  *
1052  * Return: The number of overruns are returned.
1053  */
hrtimer_forward(struct hrtimer * timer,ktime_t now,ktime_t interval)1054 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
1055 {
1056 	u64 orun = 1;
1057 	ktime_t delta;
1058 
1059 	delta = ktime_sub(now, hrtimer_get_expires(timer));
1060 
1061 	if (delta < 0)
1062 		return 0;
1063 
1064 	if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
1065 		return 0;
1066 
1067 	if (interval < hrtimer_resolution)
1068 		interval = hrtimer_resolution;
1069 
1070 	if (unlikely(delta >= interval)) {
1071 		s64 incr = ktime_to_ns(interval);
1072 
1073 		orun = ktime_divns(delta, incr);
1074 		hrtimer_add_expires_ns(timer, incr * orun);
1075 		if (hrtimer_get_expires_tv64(timer) > now)
1076 			return orun;
1077 		/*
1078 		 * This (and the ktime_add() below) is the
1079 		 * correction for exact:
1080 		 */
1081 		orun++;
1082 	}
1083 	hrtimer_add_expires(timer, interval);
1084 
1085 	return orun;
1086 }
1087 EXPORT_SYMBOL_GPL(hrtimer_forward);
1088 
1089 /*
1090  * enqueue_hrtimer - internal function to (re)start a timer
1091  *
1092  * The timer is inserted in expiry order. Insertion into the
1093  * red black tree is O(log(n)). Must hold the base lock.
1094  *
1095  * Returns true when the new timer is the leftmost timer in the tree.
1096  */
enqueue_hrtimer(struct hrtimer * timer,struct hrtimer_clock_base * base,enum hrtimer_mode mode)1097 static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1098 			    enum hrtimer_mode mode)
1099 {
1100 	debug_activate(timer, mode);
1101 	WARN_ON_ONCE(!base->cpu_base->online);
1102 
1103 	base->cpu_base->active_bases |= 1 << base->index;
1104 
1105 	/* Pairs with the lockless read in hrtimer_is_queued() */
1106 	WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED);
1107 
1108 	return timerqueue_add(&base->active, &timer->node);
1109 }
1110 
1111 /*
1112  * __remove_hrtimer - internal function to remove a timer
1113  *
1114  * Caller must hold the base lock.
1115  *
1116  * High resolution timer mode reprograms the clock event device when the
1117  * timer is the one which expires next. The caller can disable this by setting
1118  * reprogram to zero. This is useful, when the context does a reprogramming
1119  * anyway (e.g. timer interrupt)
1120  */
__remove_hrtimer(struct hrtimer * timer,struct hrtimer_clock_base * base,u8 newstate,int reprogram)1121 static void __remove_hrtimer(struct hrtimer *timer,
1122 			     struct hrtimer_clock_base *base,
1123 			     u8 newstate, int reprogram)
1124 {
1125 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1126 	u8 state = timer->state;
1127 
1128 	/* Pairs with the lockless read in hrtimer_is_queued() */
1129 	WRITE_ONCE(timer->state, newstate);
1130 	if (!(state & HRTIMER_STATE_ENQUEUED))
1131 		return;
1132 
1133 	if (!timerqueue_del(&base->active, &timer->node))
1134 		cpu_base->active_bases &= ~(1 << base->index);
1135 
1136 	/*
1137 	 * Note: If reprogram is false we do not update
1138 	 * cpu_base->next_timer. This happens when we remove the first
1139 	 * timer on a remote cpu. No harm as we never dereference
1140 	 * cpu_base->next_timer. So the worst thing what can happen is
1141 	 * an superfluous call to hrtimer_force_reprogram() on the
1142 	 * remote cpu later on if the same timer gets enqueued again.
1143 	 */
1144 	if (reprogram && timer == cpu_base->next_timer)
1145 		hrtimer_force_reprogram(cpu_base, 1);
1146 }
1147 
1148 /*
1149  * remove hrtimer, called with base lock held
1150  */
1151 static inline int
remove_hrtimer(struct hrtimer * timer,struct hrtimer_clock_base * base,bool restart,bool keep_local)1152 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1153 	       bool restart, bool keep_local)
1154 {
1155 	u8 state = timer->state;
1156 
1157 	if (state & HRTIMER_STATE_ENQUEUED) {
1158 		bool reprogram;
1159 
1160 		/*
1161 		 * Remove the timer and force reprogramming when high
1162 		 * resolution mode is active and the timer is on the current
1163 		 * CPU. If we remove a timer on another CPU, reprogramming is
1164 		 * skipped. The interrupt event on this CPU is fired and
1165 		 * reprogramming happens in the interrupt handler. This is a
1166 		 * rare case and less expensive than a smp call.
1167 		 */
1168 		debug_deactivate(timer);
1169 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1170 
1171 		/*
1172 		 * If the timer is not restarted then reprogramming is
1173 		 * required if the timer is local. If it is local and about
1174 		 * to be restarted, avoid programming it twice (on removal
1175 		 * and a moment later when it's requeued).
1176 		 */
1177 		if (!restart)
1178 			state = HRTIMER_STATE_INACTIVE;
1179 		else
1180 			reprogram &= !keep_local;
1181 
1182 		__remove_hrtimer(timer, base, state, reprogram);
1183 		return 1;
1184 	}
1185 	return 0;
1186 }
1187 
hrtimer_update_lowres(struct hrtimer * timer,ktime_t tim,const enum hrtimer_mode mode)1188 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
1189 					    const enum hrtimer_mode mode)
1190 {
1191 #ifdef CONFIG_TIME_LOW_RES
1192 	/*
1193 	 * CONFIG_TIME_LOW_RES indicates that the system has no way to return
1194 	 * granular time values. For relative timers we add hrtimer_resolution
1195 	 * (i.e. one jiffy) to prevent short timeouts.
1196 	 */
1197 	timer->is_rel = mode & HRTIMER_MODE_REL;
1198 	if (timer->is_rel)
1199 		tim = ktime_add_safe(tim, hrtimer_resolution);
1200 #endif
1201 	return tim;
1202 }
1203 
1204 static void
hrtimer_update_softirq_timer(struct hrtimer_cpu_base * cpu_base,bool reprogram)1205 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
1206 {
1207 	ktime_t expires;
1208 
1209 	/*
1210 	 * Find the next SOFT expiration.
1211 	 */
1212 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
1213 
1214 	/*
1215 	 * reprogramming needs to be triggered, even if the next soft
1216 	 * hrtimer expires at the same time than the next hard
1217 	 * hrtimer. cpu_base->softirq_expires_next needs to be updated!
1218 	 */
1219 	if (expires == KTIME_MAX)
1220 		return;
1221 
1222 	/*
1223 	 * cpu_base->*next_timer is recomputed by __hrtimer_get_next_event()
1224 	 * cpu_base->*expires_next is only set by hrtimer_reprogram()
1225 	 */
1226 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
1227 }
1228 
__hrtimer_start_range_ns(struct hrtimer * timer,ktime_t tim,u64 delta_ns,const enum hrtimer_mode mode,struct hrtimer_clock_base * base)1229 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1230 				    u64 delta_ns, const enum hrtimer_mode mode,
1231 				    struct hrtimer_clock_base *base)
1232 {
1233 	struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
1234 	struct hrtimer_clock_base *new_base;
1235 	bool force_local, first;
1236 
1237 	/*
1238 	 * If the timer is on the local cpu base and is the first expiring
1239 	 * timer then this might end up reprogramming the hardware twice
1240 	 * (on removal and on enqueue). To avoid that by prevent the
1241 	 * reprogram on removal, keep the timer local to the current CPU
1242 	 * and enforce reprogramming after it is queued no matter whether
1243 	 * it is the new first expiring timer again or not.
1244 	 */
1245 	force_local = base->cpu_base == this_cpu_base;
1246 	force_local &= base->cpu_base->next_timer == timer;
1247 
1248 	/*
1249 	 * Don't force local queuing if this enqueue happens on a unplugged
1250 	 * CPU after hrtimer_cpu_dying() has been invoked.
1251 	 */
1252 	force_local &= this_cpu_base->online;
1253 
1254 	/*
1255 	 * Remove an active timer from the queue. In case it is not queued
1256 	 * on the current CPU, make sure that remove_hrtimer() updates the
1257 	 * remote data correctly.
1258 	 *
1259 	 * If it's on the current CPU and the first expiring timer, then
1260 	 * skip reprogramming, keep the timer local and enforce
1261 	 * reprogramming later if it was the first expiring timer.  This
1262 	 * avoids programming the underlying clock event twice (once at
1263 	 * removal and once after enqueue).
1264 	 */
1265 	remove_hrtimer(timer, base, true, force_local);
1266 
1267 	if (mode & HRTIMER_MODE_REL)
1268 		tim = ktime_add_safe(tim, base->get_time());
1269 
1270 	tim = hrtimer_update_lowres(timer, tim, mode);
1271 
1272 	hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1273 
1274 	/* Switch the timer base, if necessary: */
1275 	if (!force_local) {
1276 		new_base = switch_hrtimer_base(timer, base,
1277 					       mode & HRTIMER_MODE_PINNED);
1278 	} else {
1279 		new_base = base;
1280 	}
1281 
1282 	first = enqueue_hrtimer(timer, new_base, mode);
1283 	if (!force_local) {
1284 		/*
1285 		 * If the current CPU base is online, then the timer is
1286 		 * never queued on a remote CPU if it would be the first
1287 		 * expiring timer there.
1288 		 */
1289 		if (hrtimer_base_is_online(this_cpu_base))
1290 			return first;
1291 
1292 		/*
1293 		 * Timer was enqueued remote because the current base is
1294 		 * already offline. If the timer is the first to expire,
1295 		 * kick the remote CPU to reprogram the clock event.
1296 		 */
1297 		if (first) {
1298 			struct hrtimer_cpu_base *new_cpu_base = new_base->cpu_base;
1299 
1300 			smp_call_function_single_async(new_cpu_base->cpu, &new_cpu_base->csd);
1301 		}
1302 		return 0;
1303 	}
1304 
1305 	/*
1306 	 * Timer was forced to stay on the current CPU to avoid
1307 	 * reprogramming on removal and enqueue. Force reprogram the
1308 	 * hardware by evaluating the new first expiring timer.
1309 	 */
1310 	hrtimer_force_reprogram(new_base->cpu_base, 1);
1311 	return 0;
1312 }
1313 
1314 /**
1315  * hrtimer_start_range_ns - (re)start an hrtimer
1316  * @timer:	the timer to be added
1317  * @tim:	expiry time
1318  * @delta_ns:	"slack" range for the timer
1319  * @mode:	timer mode: absolute (HRTIMER_MODE_ABS) or
1320  *		relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
1321  *		softirq based mode is considered for debug purpose only!
1322  */
hrtimer_start_range_ns(struct hrtimer * timer,ktime_t tim,u64 delta_ns,const enum hrtimer_mode mode)1323 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1324 			    u64 delta_ns, const enum hrtimer_mode mode)
1325 {
1326 	struct hrtimer_clock_base *base;
1327 	unsigned long flags;
1328 
1329 	if (WARN_ON_ONCE(!timer->function))
1330 		return;
1331 	/*
1332 	 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
1333 	 * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
1334 	 * expiry mode because unmarked timers are moved to softirq expiry.
1335 	 */
1336 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1337 		WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1338 	else
1339 		WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
1340 
1341 	base = lock_hrtimer_base(timer, &flags);
1342 
1343 	if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
1344 		hrtimer_reprogram(timer, true);
1345 
1346 	unlock_hrtimer_base(timer, &flags);
1347 }
1348 EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
1349 
1350 /**
1351  * hrtimer_try_to_cancel - try to deactivate a timer
1352  * @timer:	hrtimer to stop
1353  *
1354  * Returns:
1355  *
1356  *  *  0 when the timer was not active
1357  *  *  1 when the timer was active
1358  *  * -1 when the timer is currently executing the callback function and
1359  *    cannot be stopped
1360  */
hrtimer_try_to_cancel(struct hrtimer * timer)1361 int hrtimer_try_to_cancel(struct hrtimer *timer)
1362 {
1363 	struct hrtimer_clock_base *base;
1364 	unsigned long flags;
1365 	int ret = -1;
1366 
1367 	/*
1368 	 * Check lockless first. If the timer is not active (neither
1369 	 * enqueued nor running the callback, nothing to do here.  The
1370 	 * base lock does not serialize against a concurrent enqueue,
1371 	 * so we can avoid taking it.
1372 	 */
1373 	if (!hrtimer_active(timer))
1374 		return 0;
1375 
1376 	base = lock_hrtimer_base(timer, &flags);
1377 
1378 	if (!hrtimer_callback_running(timer))
1379 		ret = remove_hrtimer(timer, base, false, false);
1380 
1381 	unlock_hrtimer_base(timer, &flags);
1382 
1383 	return ret;
1384 
1385 }
1386 EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
1387 
1388 #ifdef CONFIG_PREEMPT_RT
hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base * base)1389 static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
1390 {
1391 	spin_lock_init(&base->softirq_expiry_lock);
1392 }
1393 
hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base * base)1394 static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
1395 	__acquires(&base->softirq_expiry_lock)
1396 {
1397 	spin_lock(&base->softirq_expiry_lock);
1398 }
1399 
hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base * base)1400 static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
1401 	__releases(&base->softirq_expiry_lock)
1402 {
1403 	spin_unlock(&base->softirq_expiry_lock);
1404 }
1405 
1406 /*
1407  * The counterpart to hrtimer_cancel_wait_running().
1408  *
1409  * If there is a waiter for cpu_base->expiry_lock, then it was waiting for
1410  * the timer callback to finish. Drop expiry_lock and reacquire it. That
1411  * allows the waiter to acquire the lock and make progress.
1412  */
hrtimer_sync_wait_running(struct hrtimer_cpu_base * cpu_base,unsigned long flags)1413 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
1414 				      unsigned long flags)
1415 {
1416 	if (atomic_read(&cpu_base->timer_waiters)) {
1417 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1418 		spin_unlock(&cpu_base->softirq_expiry_lock);
1419 		spin_lock(&cpu_base->softirq_expiry_lock);
1420 		raw_spin_lock_irq(&cpu_base->lock);
1421 	}
1422 }
1423 
1424 #ifdef CONFIG_SMP
is_migration_base(struct hrtimer_clock_base * base)1425 static __always_inline bool is_migration_base(struct hrtimer_clock_base *base)
1426 {
1427 	return base == &migration_base;
1428 }
1429 #else
is_migration_base(struct hrtimer_clock_base * base)1430 static __always_inline bool is_migration_base(struct hrtimer_clock_base *base)
1431 {
1432 	return false;
1433 }
1434 #endif
1435 
1436 /*
1437  * This function is called on PREEMPT_RT kernels when the fast path
1438  * deletion of a timer failed because the timer callback function was
1439  * running.
1440  *
1441  * This prevents priority inversion: if the soft irq thread is preempted
1442  * in the middle of a timer callback, then calling del_timer_sync() can
1443  * lead to two issues:
1444  *
1445  *  - If the caller is on a remote CPU then it has to spin wait for the timer
1446  *    handler to complete. This can result in unbound priority inversion.
1447  *
1448  *  - If the caller originates from the task which preempted the timer
1449  *    handler on the same CPU, then spin waiting for the timer handler to
1450  *    complete is never going to end.
1451  */
hrtimer_cancel_wait_running(const struct hrtimer * timer)1452 void hrtimer_cancel_wait_running(const struct hrtimer *timer)
1453 {
1454 	/* Lockless read. Prevent the compiler from reloading it below */
1455 	struct hrtimer_clock_base *base = READ_ONCE(timer->base);
1456 
1457 	/*
1458 	 * Just relax if the timer expires in hard interrupt context or if
1459 	 * it is currently on the migration base.
1460 	 */
1461 	if (!timer->is_soft || is_migration_base(base)) {
1462 		cpu_relax();
1463 		return;
1464 	}
1465 
1466 	/*
1467 	 * Mark the base as contended and grab the expiry lock, which is
1468 	 * held by the softirq across the timer callback. Drop the lock
1469 	 * immediately so the softirq can expire the next timer. In theory
1470 	 * the timer could already be running again, but that's more than
1471 	 * unlikely and just causes another wait loop.
1472 	 */
1473 	atomic_inc(&base->cpu_base->timer_waiters);
1474 	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
1475 	atomic_dec(&base->cpu_base->timer_waiters);
1476 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
1477 }
1478 #else
1479 static inline void
hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base * base)1480 hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
1481 static inline void
hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base * base)1482 hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
1483 static inline void
hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base * base)1484 hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
hrtimer_sync_wait_running(struct hrtimer_cpu_base * base,unsigned long flags)1485 static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
1486 					     unsigned long flags) { }
1487 #endif
1488 
1489 /**
1490  * hrtimer_cancel - cancel a timer and wait for the handler to finish.
1491  * @timer:	the timer to be cancelled
1492  *
1493  * Returns:
1494  *  0 when the timer was not active
1495  *  1 when the timer was active
1496  */
hrtimer_cancel(struct hrtimer * timer)1497 int hrtimer_cancel(struct hrtimer *timer)
1498 {
1499 	int ret;
1500 
1501 	do {
1502 		ret = hrtimer_try_to_cancel(timer);
1503 
1504 		if (ret < 0)
1505 			hrtimer_cancel_wait_running(timer);
1506 	} while (ret < 0);
1507 	return ret;
1508 }
1509 EXPORT_SYMBOL_GPL(hrtimer_cancel);
1510 
1511 /**
1512  * __hrtimer_get_remaining - get remaining time for the timer
1513  * @timer:	the timer to read
1514  * @adjust:	adjust relative timers when CONFIG_TIME_LOW_RES=y
1515  */
__hrtimer_get_remaining(const struct hrtimer * timer,bool adjust)1516 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
1517 {
1518 	unsigned long flags;
1519 	ktime_t rem;
1520 
1521 	lock_hrtimer_base(timer, &flags);
1522 	if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
1523 		rem = hrtimer_expires_remaining_adjusted(timer);
1524 	else
1525 		rem = hrtimer_expires_remaining(timer);
1526 	unlock_hrtimer_base(timer, &flags);
1527 
1528 	return rem;
1529 }
1530 EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
1531 
1532 #ifdef CONFIG_NO_HZ_COMMON
1533 /**
1534  * hrtimer_get_next_event - get the time until next expiry event
1535  *
1536  * Returns the next expiry time or KTIME_MAX if no timer is pending.
1537  */
hrtimer_get_next_event(void)1538 u64 hrtimer_get_next_event(void)
1539 {
1540 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1541 	u64 expires = KTIME_MAX;
1542 	unsigned long flags;
1543 
1544 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1545 
1546 	if (!hrtimer_hres_active(cpu_base))
1547 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1548 
1549 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1550 
1551 	return expires;
1552 }
1553 
1554 /**
1555  * hrtimer_next_event_without - time until next expiry event w/o one timer
1556  * @exclude:	timer to exclude
1557  *
1558  * Returns the next expiry time over all timers except for the @exclude one or
1559  * KTIME_MAX if none of them is pending.
1560  */
hrtimer_next_event_without(const struct hrtimer * exclude)1561 u64 hrtimer_next_event_without(const struct hrtimer *exclude)
1562 {
1563 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1564 	u64 expires = KTIME_MAX;
1565 	unsigned long flags;
1566 
1567 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1568 
1569 	if (hrtimer_hres_active(cpu_base)) {
1570 		unsigned int active;
1571 
1572 		if (!cpu_base->softirq_activated) {
1573 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1574 			expires = __hrtimer_next_event_base(cpu_base, exclude,
1575 							    active, KTIME_MAX);
1576 		}
1577 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1578 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,
1579 						    expires);
1580 	}
1581 
1582 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1583 
1584 	return expires;
1585 }
1586 #endif
1587 
hrtimer_clockid_to_base(clockid_t clock_id)1588 static inline int hrtimer_clockid_to_base(clockid_t clock_id)
1589 {
1590 	if (likely(clock_id < MAX_CLOCKS)) {
1591 		int base = hrtimer_clock_to_base_table[clock_id];
1592 
1593 		if (likely(base != HRTIMER_MAX_CLOCK_BASES))
1594 			return base;
1595 	}
1596 	WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
1597 	return HRTIMER_BASE_MONOTONIC;
1598 }
1599 
hrtimer_dummy_timeout(struct hrtimer * unused)1600 static enum hrtimer_restart hrtimer_dummy_timeout(struct hrtimer *unused)
1601 {
1602 	return HRTIMER_NORESTART;
1603 }
1604 
__hrtimer_init(struct hrtimer * timer,clockid_t clock_id,enum hrtimer_mode mode)1605 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1606 			   enum hrtimer_mode mode)
1607 {
1608 	bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1609 	struct hrtimer_cpu_base *cpu_base;
1610 	int base;
1611 
1612 	/*
1613 	 * On PREEMPT_RT enabled kernels hrtimers which are not explicitly
1614 	 * marked for hard interrupt expiry mode are moved into soft
1615 	 * interrupt context for latency reasons and because the callbacks
1616 	 * can invoke functions which might sleep on RT, e.g. spin_lock().
1617 	 */
1618 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
1619 		softtimer = true;
1620 
1621 	memset(timer, 0, sizeof(struct hrtimer));
1622 
1623 	cpu_base = raw_cpu_ptr(&hrtimer_bases);
1624 
1625 	/*
1626 	 * POSIX magic: Relative CLOCK_REALTIME timers are not affected by
1627 	 * clock modifications, so they needs to become CLOCK_MONOTONIC to
1628 	 * ensure POSIX compliance.
1629 	 */
1630 	if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
1631 		clock_id = CLOCK_MONOTONIC;
1632 
1633 	base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
1634 	base += hrtimer_clockid_to_base(clock_id);
1635 	timer->is_soft = softtimer;
1636 	timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
1637 	timer->base = &cpu_base->clock_base[base];
1638 	timerqueue_init(&timer->node);
1639 }
1640 
__hrtimer_setup(struct hrtimer * timer,enum hrtimer_restart (* function)(struct hrtimer *),clockid_t clock_id,enum hrtimer_mode mode)1641 static void __hrtimer_setup(struct hrtimer *timer,
1642 			    enum hrtimer_restart (*function)(struct hrtimer *),
1643 			    clockid_t clock_id, enum hrtimer_mode mode)
1644 {
1645 	__hrtimer_init(timer, clock_id, mode);
1646 
1647 	if (WARN_ON_ONCE(!function))
1648 		timer->function = hrtimer_dummy_timeout;
1649 	else
1650 		timer->function = function;
1651 }
1652 
1653 /**
1654  * hrtimer_init - initialize a timer to the given clock
1655  * @timer:	the timer to be initialized
1656  * @clock_id:	the clock to be used
1657  * @mode:       The modes which are relevant for initialization:
1658  *              HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
1659  *              HRTIMER_MODE_REL_SOFT
1660  *
1661  *              The PINNED variants of the above can be handed in,
1662  *              but the PINNED bit is ignored as pinning happens
1663  *              when the hrtimer is started
1664  */
hrtimer_init(struct hrtimer * timer,clockid_t clock_id,enum hrtimer_mode mode)1665 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1666 		  enum hrtimer_mode mode)
1667 {
1668 	debug_init(timer, clock_id, mode);
1669 	__hrtimer_init(timer, clock_id, mode);
1670 }
1671 EXPORT_SYMBOL_GPL(hrtimer_init);
1672 
1673 /**
1674  * hrtimer_setup - initialize a timer to the given clock
1675  * @timer:	the timer to be initialized
1676  * @function:	the callback function
1677  * @clock_id:	the clock to be used
1678  * @mode:       The modes which are relevant for initialization:
1679  *              HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
1680  *              HRTIMER_MODE_REL_SOFT
1681  *
1682  *              The PINNED variants of the above can be handed in,
1683  *              but the PINNED bit is ignored as pinning happens
1684  *              when the hrtimer is started
1685  */
hrtimer_setup(struct hrtimer * timer,enum hrtimer_restart (* function)(struct hrtimer *),clockid_t clock_id,enum hrtimer_mode mode)1686 void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),
1687 		   clockid_t clock_id, enum hrtimer_mode mode)
1688 {
1689 	debug_init(timer, clock_id, mode);
1690 	__hrtimer_setup(timer, function, clock_id, mode);
1691 }
1692 EXPORT_SYMBOL_GPL(hrtimer_setup);
1693 
1694 /**
1695  * hrtimer_setup_on_stack - initialize a timer on stack memory
1696  * @timer:	The timer to be initialized
1697  * @function:	the callback function
1698  * @clock_id:	The clock to be used
1699  * @mode:       The timer mode
1700  *
1701  * Similar to hrtimer_setup(), except that this one must be used if struct hrtimer is in stack
1702  * memory.
1703  */
hrtimer_setup_on_stack(struct hrtimer * timer,enum hrtimer_restart (* function)(struct hrtimer *),clockid_t clock_id,enum hrtimer_mode mode)1704 void hrtimer_setup_on_stack(struct hrtimer *timer,
1705 			    enum hrtimer_restart (*function)(struct hrtimer *),
1706 			    clockid_t clock_id, enum hrtimer_mode mode)
1707 {
1708 	debug_init_on_stack(timer, clock_id, mode);
1709 	__hrtimer_setup(timer, function, clock_id, mode);
1710 }
1711 EXPORT_SYMBOL_GPL(hrtimer_setup_on_stack);
1712 
1713 /*
1714  * A timer is active, when it is enqueued into the rbtree or the
1715  * callback function is running or it's in the state of being migrated
1716  * to another cpu.
1717  *
1718  * It is important for this function to not return a false negative.
1719  */
hrtimer_active(const struct hrtimer * timer)1720 bool hrtimer_active(const struct hrtimer *timer)
1721 {
1722 	struct hrtimer_clock_base *base;
1723 	unsigned int seq;
1724 
1725 	do {
1726 		base = READ_ONCE(timer->base);
1727 		seq = raw_read_seqcount_begin(&base->seq);
1728 
1729 		if (timer->state != HRTIMER_STATE_INACTIVE ||
1730 		    base->running == timer)
1731 			return true;
1732 
1733 	} while (read_seqcount_retry(&base->seq, seq) ||
1734 		 base != READ_ONCE(timer->base));
1735 
1736 	return false;
1737 }
1738 EXPORT_SYMBOL_GPL(hrtimer_active);
1739 
1740 /*
1741  * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
1742  * distinct sections:
1743  *
1744  *  - queued:	the timer is queued
1745  *  - callback:	the timer is being ran
1746  *  - post:	the timer is inactive or (re)queued
1747  *
1748  * On the read side we ensure we observe timer->state and cpu_base->running
1749  * from the same section, if anything changed while we looked at it, we retry.
1750  * This includes timer->base changing because sequence numbers alone are
1751  * insufficient for that.
1752  *
1753  * The sequence numbers are required because otherwise we could still observe
1754  * a false negative if the read side got smeared over multiple consecutive
1755  * __run_hrtimer() invocations.
1756  */
1757 
__run_hrtimer(struct hrtimer_cpu_base * cpu_base,struct hrtimer_clock_base * base,struct hrtimer * timer,ktime_t * now,unsigned long flags)1758 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1759 			  struct hrtimer_clock_base *base,
1760 			  struct hrtimer *timer, ktime_t *now,
1761 			  unsigned long flags) __must_hold(&cpu_base->lock)
1762 {
1763 	enum hrtimer_restart (*fn)(struct hrtimer *);
1764 	bool expires_in_hardirq;
1765 	int restart;
1766 
1767 	lockdep_assert_held(&cpu_base->lock);
1768 
1769 	debug_deactivate(timer);
1770 	base->running = timer;
1771 
1772 	/*
1773 	 * Separate the ->running assignment from the ->state assignment.
1774 	 *
1775 	 * As with a regular write barrier, this ensures the read side in
1776 	 * hrtimer_active() cannot observe base->running == NULL &&
1777 	 * timer->state == INACTIVE.
1778 	 */
1779 	raw_write_seqcount_barrier(&base->seq);
1780 
1781 	__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
1782 	fn = timer->function;
1783 
1784 	/*
1785 	 * Clear the 'is relative' flag for the TIME_LOW_RES case. If the
1786 	 * timer is restarted with a period then it becomes an absolute
1787 	 * timer. If its not restarted it does not matter.
1788 	 */
1789 	if (IS_ENABLED(CONFIG_TIME_LOW_RES))
1790 		timer->is_rel = false;
1791 
1792 	/*
1793 	 * The timer is marked as running in the CPU base, so it is
1794 	 * protected against migration to a different CPU even if the lock
1795 	 * is dropped.
1796 	 */
1797 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1798 	trace_hrtimer_expire_entry(timer, now);
1799 	expires_in_hardirq = lockdep_hrtimer_enter(timer);
1800 
1801 	restart = fn(timer);
1802 
1803 	lockdep_hrtimer_exit(expires_in_hardirq);
1804 	trace_hrtimer_expire_exit(timer);
1805 	raw_spin_lock_irq(&cpu_base->lock);
1806 
1807 	/*
1808 	 * Note: We clear the running state after enqueue_hrtimer and
1809 	 * we do not reprogram the event hardware. Happens either in
1810 	 * hrtimer_start_range_ns() or in hrtimer_interrupt()
1811 	 *
1812 	 * Note: Because we dropped the cpu_base->lock above,
1813 	 * hrtimer_start_range_ns() can have popped in and enqueued the timer
1814 	 * for us already.
1815 	 */
1816 	if (restart != HRTIMER_NORESTART &&
1817 	    !(timer->state & HRTIMER_STATE_ENQUEUED))
1818 		enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
1819 
1820 	/*
1821 	 * Separate the ->running assignment from the ->state assignment.
1822 	 *
1823 	 * As with a regular write barrier, this ensures the read side in
1824 	 * hrtimer_active() cannot observe base->running.timer == NULL &&
1825 	 * timer->state == INACTIVE.
1826 	 */
1827 	raw_write_seqcount_barrier(&base->seq);
1828 
1829 	WARN_ON_ONCE(base->running != timer);
1830 	base->running = NULL;
1831 }
1832 
__hrtimer_run_queues(struct hrtimer_cpu_base * cpu_base,ktime_t now,unsigned long flags,unsigned int active_mask)1833 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
1834 				 unsigned long flags, unsigned int active_mask)
1835 {
1836 	struct hrtimer_clock_base *base;
1837 	unsigned int active = cpu_base->active_bases & active_mask;
1838 
1839 	for_each_active_base(base, cpu_base, active) {
1840 		struct timerqueue_node *node;
1841 		ktime_t basenow;
1842 
1843 		basenow = ktime_add(now, base->offset);
1844 
1845 		while ((node = timerqueue_getnext(&base->active))) {
1846 			struct hrtimer *timer;
1847 
1848 			timer = container_of(node, struct hrtimer, node);
1849 
1850 			/*
1851 			 * The immediate goal for using the softexpires is
1852 			 * minimizing wakeups, not running timers at the
1853 			 * earliest interrupt after their soft expiration.
1854 			 * This allows us to avoid using a Priority Search
1855 			 * Tree, which can answer a stabbing query for
1856 			 * overlapping intervals and instead use the simple
1857 			 * BST we already have.
1858 			 * We don't add extra wakeups by delaying timers that
1859 			 * are right-of a not yet expired timer, because that
1860 			 * timer will have to trigger a wakeup anyway.
1861 			 */
1862 			if (basenow < hrtimer_get_softexpires_tv64(timer))
1863 				break;
1864 
1865 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);
1866 			if (active_mask == HRTIMER_ACTIVE_SOFT)
1867 				hrtimer_sync_wait_running(cpu_base, flags);
1868 		}
1869 	}
1870 }
1871 
hrtimer_run_softirq(void)1872 static __latent_entropy void hrtimer_run_softirq(void)
1873 {
1874 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1875 	unsigned long flags;
1876 	ktime_t now;
1877 
1878 	hrtimer_cpu_base_lock_expiry(cpu_base);
1879 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1880 
1881 	now = hrtimer_update_base(cpu_base);
1882 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
1883 
1884 	cpu_base->softirq_activated = 0;
1885 	hrtimer_update_softirq_timer(cpu_base, true);
1886 
1887 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1888 	hrtimer_cpu_base_unlock_expiry(cpu_base);
1889 }
1890 
1891 #ifdef CONFIG_HIGH_RES_TIMERS
1892 
1893 /*
1894  * High resolution timer interrupt
1895  * Called with interrupts disabled
1896  */
hrtimer_interrupt(struct clock_event_device * dev)1897 void hrtimer_interrupt(struct clock_event_device *dev)
1898 {
1899 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1900 	ktime_t expires_next, now, entry_time, delta;
1901 	unsigned long flags;
1902 	int retries = 0;
1903 
1904 	BUG_ON(!cpu_base->hres_active);
1905 	cpu_base->nr_events++;
1906 	dev->next_event = KTIME_MAX;
1907 
1908 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1909 	entry_time = now = hrtimer_update_base(cpu_base);
1910 retry:
1911 	cpu_base->in_hrtirq = 1;
1912 	/*
1913 	 * We set expires_next to KTIME_MAX here with cpu_base->lock
1914 	 * held to prevent that a timer is enqueued in our queue via
1915 	 * the migration code. This does not affect enqueueing of
1916 	 * timers which run their callback and need to be requeued on
1917 	 * this CPU.
1918 	 */
1919 	cpu_base->expires_next = KTIME_MAX;
1920 
1921 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1922 		cpu_base->softirq_expires_next = KTIME_MAX;
1923 		cpu_base->softirq_activated = 1;
1924 		raise_timer_softirq(HRTIMER_SOFTIRQ);
1925 	}
1926 
1927 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1928 
1929 	/* Reevaluate the clock bases for the [soft] next expiry */
1930 	expires_next = hrtimer_update_next_event(cpu_base);
1931 	/*
1932 	 * Store the new expiry value so the migration code can verify
1933 	 * against it.
1934 	 */
1935 	cpu_base->expires_next = expires_next;
1936 	cpu_base->in_hrtirq = 0;
1937 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1938 
1939 	/* Reprogramming necessary ? */
1940 	if (!tick_program_event(expires_next, 0)) {
1941 		cpu_base->hang_detected = 0;
1942 		return;
1943 	}
1944 
1945 	/*
1946 	 * The next timer was already expired due to:
1947 	 * - tracing
1948 	 * - long lasting callbacks
1949 	 * - being scheduled away when running in a VM
1950 	 *
1951 	 * We need to prevent that we loop forever in the hrtimer
1952 	 * interrupt routine. We give it 3 attempts to avoid
1953 	 * overreacting on some spurious event.
1954 	 *
1955 	 * Acquire base lock for updating the offsets and retrieving
1956 	 * the current time.
1957 	 */
1958 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
1959 	now = hrtimer_update_base(cpu_base);
1960 	cpu_base->nr_retries++;
1961 	if (++retries < 3)
1962 		goto retry;
1963 	/*
1964 	 * Give the system a chance to do something else than looping
1965 	 * here. We stored the entry time, so we know exactly how long
1966 	 * we spent here. We schedule the next event this amount of
1967 	 * time away.
1968 	 */
1969 	cpu_base->nr_hangs++;
1970 	cpu_base->hang_detected = 1;
1971 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1972 
1973 	delta = ktime_sub(now, entry_time);
1974 	if ((unsigned int)delta > cpu_base->max_hang_time)
1975 		cpu_base->max_hang_time = (unsigned int) delta;
1976 	/*
1977 	 * Limit it to a sensible value as we enforce a longer
1978 	 * delay. Give the CPU at least 100ms to catch up.
1979 	 */
1980 	if (delta > 100 * NSEC_PER_MSEC)
1981 		expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
1982 	else
1983 		expires_next = ktime_add(now, delta);
1984 	tick_program_event(expires_next, 1);
1985 	pr_warn_once("hrtimer: interrupt took %llu ns\n", ktime_to_ns(delta));
1986 }
1987 #endif /* !CONFIG_HIGH_RES_TIMERS */
1988 
1989 /*
1990  * Called from run_local_timers in hardirq context every jiffy
1991  */
hrtimer_run_queues(void)1992 void hrtimer_run_queues(void)
1993 {
1994 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1995 	unsigned long flags;
1996 	ktime_t now;
1997 
1998 	if (hrtimer_hres_active(cpu_base))
1999 		return;
2000 
2001 	/*
2002 	 * This _is_ ugly: We have to check periodically, whether we
2003 	 * can switch to highres and / or nohz mode. The clocksource
2004 	 * switch happens with xtime_lock held. Notification from
2005 	 * there only sets the check bit in the tick_oneshot code,
2006 	 * otherwise we might deadlock vs. xtime_lock.
2007 	 */
2008 	if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) {
2009 		hrtimer_switch_to_hres();
2010 		return;
2011 	}
2012 
2013 	raw_spin_lock_irqsave(&cpu_base->lock, flags);
2014 	now = hrtimer_update_base(cpu_base);
2015 
2016 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {
2017 		cpu_base->softirq_expires_next = KTIME_MAX;
2018 		cpu_base->softirq_activated = 1;
2019 		raise_timer_softirq(HRTIMER_SOFTIRQ);
2020 	}
2021 
2022 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
2023 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2024 }
2025 
2026 /*
2027  * Sleep related functions:
2028  */
hrtimer_wakeup(struct hrtimer * timer)2029 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
2030 {
2031 	struct hrtimer_sleeper *t =
2032 		container_of(timer, struct hrtimer_sleeper, timer);
2033 	struct task_struct *task = t->task;
2034 
2035 	t->task = NULL;
2036 	if (task)
2037 		wake_up_process(task);
2038 
2039 	return HRTIMER_NORESTART;
2040 }
2041 
2042 /**
2043  * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
2044  * @sl:		sleeper to be started
2045  * @mode:	timer mode abs/rel
2046  *
2047  * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
2048  * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
2049  */
hrtimer_sleeper_start_expires(struct hrtimer_sleeper * sl,enum hrtimer_mode mode)2050 void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
2051 				   enum hrtimer_mode mode)
2052 {
2053 	/*
2054 	 * Make the enqueue delivery mode check work on RT. If the sleeper
2055 	 * was initialized for hard interrupt delivery, force the mode bit.
2056 	 * This is a special case for hrtimer_sleepers because
2057 	 * __hrtimer_init_sleeper() determines the delivery mode on RT so the
2058 	 * fiddling with this decision is avoided at the call sites.
2059 	 */
2060 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
2061 		mode |= HRTIMER_MODE_HARD;
2062 
2063 	hrtimer_start_expires(&sl->timer, mode);
2064 }
2065 EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
2066 
__hrtimer_init_sleeper(struct hrtimer_sleeper * sl,clockid_t clock_id,enum hrtimer_mode mode)2067 static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
2068 				   clockid_t clock_id, enum hrtimer_mode mode)
2069 {
2070 	/*
2071 	 * On PREEMPT_RT enabled kernels hrtimers which are not explicitly
2072 	 * marked for hard interrupt expiry mode are moved into soft
2073 	 * interrupt context either for latency reasons or because the
2074 	 * hrtimer callback takes regular spinlocks or invokes other
2075 	 * functions which are not suitable for hard interrupt context on
2076 	 * PREEMPT_RT.
2077 	 *
2078 	 * The hrtimer_sleeper callback is RT compatible in hard interrupt
2079 	 * context, but there is a latency concern: Untrusted userspace can
2080 	 * spawn many threads which arm timers for the same expiry time on
2081 	 * the same CPU. That causes a latency spike due to the wakeup of
2082 	 * a gazillion threads.
2083 	 *
2084 	 * OTOH, privileged real-time user space applications rely on the
2085 	 * low latency of hard interrupt wakeups. If the current task is in
2086 	 * a real-time scheduling class, mark the mode for hard interrupt
2087 	 * expiry.
2088 	 */
2089 	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
2090 		if (rt_or_dl_task_policy(current) && !(mode & HRTIMER_MODE_SOFT))
2091 			mode |= HRTIMER_MODE_HARD;
2092 	}
2093 
2094 	__hrtimer_init(&sl->timer, clock_id, mode);
2095 	sl->timer.function = hrtimer_wakeup;
2096 	sl->task = current;
2097 }
2098 
2099 /**
2100  * hrtimer_setup_sleeper_on_stack - initialize a sleeper in stack memory
2101  * @sl:		sleeper to be initialized
2102  * @clock_id:	the clock to be used
2103  * @mode:	timer mode abs/rel
2104  */
hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper * sl,clockid_t clock_id,enum hrtimer_mode mode)2105 void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl,
2106 				    clockid_t clock_id, enum hrtimer_mode mode)
2107 {
2108 	debug_init_on_stack(&sl->timer, clock_id, mode);
2109 	__hrtimer_init_sleeper(sl, clock_id, mode);
2110 }
2111 EXPORT_SYMBOL_GPL(hrtimer_setup_sleeper_on_stack);
2112 
nanosleep_copyout(struct restart_block * restart,struct timespec64 * ts)2113 int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
2114 {
2115 	switch(restart->nanosleep.type) {
2116 #ifdef CONFIG_COMPAT_32BIT_TIME
2117 	case TT_COMPAT:
2118 		if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
2119 			return -EFAULT;
2120 		break;
2121 #endif
2122 	case TT_NATIVE:
2123 		if (put_timespec64(ts, restart->nanosleep.rmtp))
2124 			return -EFAULT;
2125 		break;
2126 	default:
2127 		BUG();
2128 	}
2129 	return -ERESTART_RESTARTBLOCK;
2130 }
2131 
do_nanosleep(struct hrtimer_sleeper * t,enum hrtimer_mode mode)2132 static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
2133 {
2134 	struct restart_block *restart;
2135 
2136 	do {
2137 		set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
2138 		hrtimer_sleeper_start_expires(t, mode);
2139 
2140 		if (likely(t->task))
2141 			schedule();
2142 
2143 		hrtimer_cancel(&t->timer);
2144 		mode = HRTIMER_MODE_ABS;
2145 
2146 	} while (t->task && !signal_pending(current));
2147 
2148 	__set_current_state(TASK_RUNNING);
2149 
2150 	if (!t->task)
2151 		return 0;
2152 
2153 	restart = &current->restart_block;
2154 	if (restart->nanosleep.type != TT_NONE) {
2155 		ktime_t rem = hrtimer_expires_remaining(&t->timer);
2156 		struct timespec64 rmt;
2157 
2158 		if (rem <= 0)
2159 			return 0;
2160 		rmt = ktime_to_timespec64(rem);
2161 
2162 		return nanosleep_copyout(restart, &rmt);
2163 	}
2164 	return -ERESTART_RESTARTBLOCK;
2165 }
2166 
hrtimer_nanosleep_restart(struct restart_block * restart)2167 static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
2168 {
2169 	struct hrtimer_sleeper t;
2170 	int ret;
2171 
2172 	hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS);
2173 	hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
2174 	ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
2175 	destroy_hrtimer_on_stack(&t.timer);
2176 	return ret;
2177 }
2178 
hrtimer_nanosleep(ktime_t rqtp,const enum hrtimer_mode mode,const clockid_t clockid)2179 long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
2180 		       const clockid_t clockid)
2181 {
2182 	struct restart_block *restart;
2183 	struct hrtimer_sleeper t;
2184 	int ret = 0;
2185 
2186 	hrtimer_setup_sleeper_on_stack(&t, clockid, mode);
2187 	hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns);
2188 	ret = do_nanosleep(&t, mode);
2189 	if (ret != -ERESTART_RESTARTBLOCK)
2190 		goto out;
2191 
2192 	/* Absolute timers do not update the rmtp value and restart: */
2193 	if (mode == HRTIMER_MODE_ABS) {
2194 		ret = -ERESTARTNOHAND;
2195 		goto out;
2196 	}
2197 
2198 	restart = &current->restart_block;
2199 	restart->nanosleep.clockid = t.timer.base->clockid;
2200 	restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
2201 	set_restart_fn(restart, hrtimer_nanosleep_restart);
2202 out:
2203 	destroy_hrtimer_on_stack(&t.timer);
2204 	return ret;
2205 }
2206 
2207 #ifdef CONFIG_64BIT
2208 
SYSCALL_DEFINE2(nanosleep,struct __kernel_timespec __user *,rqtp,struct __kernel_timespec __user *,rmtp)2209 SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
2210 		struct __kernel_timespec __user *, rmtp)
2211 {
2212 	struct timespec64 tu;
2213 
2214 	if (get_timespec64(&tu, rqtp))
2215 		return -EFAULT;
2216 
2217 	if (!timespec64_valid(&tu))
2218 		return -EINVAL;
2219 
2220 	current->restart_block.fn = do_no_restart_syscall;
2221 	current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
2222 	current->restart_block.nanosleep.rmtp = rmtp;
2223 	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
2224 				 CLOCK_MONOTONIC);
2225 }
2226 
2227 #endif
2228 
2229 #ifdef CONFIG_COMPAT_32BIT_TIME
2230 
SYSCALL_DEFINE2(nanosleep_time32,struct old_timespec32 __user *,rqtp,struct old_timespec32 __user *,rmtp)2231 SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
2232 		       struct old_timespec32 __user *, rmtp)
2233 {
2234 	struct timespec64 tu;
2235 
2236 	if (get_old_timespec32(&tu, rqtp))
2237 		return -EFAULT;
2238 
2239 	if (!timespec64_valid(&tu))
2240 		return -EINVAL;
2241 
2242 	current->restart_block.fn = do_no_restart_syscall;
2243 	current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
2244 	current->restart_block.nanosleep.compat_rmtp = rmtp;
2245 	return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL,
2246 				 CLOCK_MONOTONIC);
2247 }
2248 #endif
2249 
2250 /*
2251  * Functions related to boot-time initialization:
2252  */
hrtimers_prepare_cpu(unsigned int cpu)2253 int hrtimers_prepare_cpu(unsigned int cpu)
2254 {
2255 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
2256 	int i;
2257 
2258 	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
2259 		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
2260 
2261 		clock_b->cpu_base = cpu_base;
2262 		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
2263 		timerqueue_init_head(&clock_b->active);
2264 	}
2265 
2266 	cpu_base->cpu = cpu;
2267 	hrtimer_cpu_base_init_expiry_lock(cpu_base);
2268 	return 0;
2269 }
2270 
hrtimers_cpu_starting(unsigned int cpu)2271 int hrtimers_cpu_starting(unsigned int cpu)
2272 {
2273 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2274 
2275 	/* Clear out any left over state from a CPU down operation */
2276 	cpu_base->active_bases = 0;
2277 	cpu_base->hres_active = 0;
2278 	cpu_base->hang_detected = 0;
2279 	cpu_base->next_timer = NULL;
2280 	cpu_base->softirq_next_timer = NULL;
2281 	cpu_base->expires_next = KTIME_MAX;
2282 	cpu_base->softirq_expires_next = KTIME_MAX;
2283 	cpu_base->online = 1;
2284 	return 0;
2285 }
2286 
2287 #ifdef CONFIG_HOTPLUG_CPU
2288 
migrate_hrtimer_list(struct hrtimer_clock_base * old_base,struct hrtimer_clock_base * new_base)2289 static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
2290 				struct hrtimer_clock_base *new_base)
2291 {
2292 	struct hrtimer *timer;
2293 	struct timerqueue_node *node;
2294 
2295 	while ((node = timerqueue_getnext(&old_base->active))) {
2296 		timer = container_of(node, struct hrtimer, node);
2297 		BUG_ON(hrtimer_callback_running(timer));
2298 		debug_deactivate(timer);
2299 
2300 		/*
2301 		 * Mark it as ENQUEUED not INACTIVE otherwise the
2302 		 * timer could be seen as !active and just vanish away
2303 		 * under us on another CPU
2304 		 */
2305 		__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
2306 		timer->base = new_base;
2307 		/*
2308 		 * Enqueue the timers on the new cpu. This does not
2309 		 * reprogram the event device in case the timer
2310 		 * expires before the earliest on this CPU, but we run
2311 		 * hrtimer_interrupt after we migrated everything to
2312 		 * sort out already expired timers and reprogram the
2313 		 * event device.
2314 		 */
2315 		enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
2316 	}
2317 }
2318 
hrtimers_cpu_dying(unsigned int dying_cpu)2319 int hrtimers_cpu_dying(unsigned int dying_cpu)
2320 {
2321 	int i, ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER));
2322 	struct hrtimer_cpu_base *old_base, *new_base;
2323 
2324 	old_base = this_cpu_ptr(&hrtimer_bases);
2325 	new_base = &per_cpu(hrtimer_bases, ncpu);
2326 
2327 	/*
2328 	 * The caller is globally serialized and nobody else
2329 	 * takes two locks at once, deadlock is not possible.
2330 	 */
2331 	raw_spin_lock(&old_base->lock);
2332 	raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING);
2333 
2334 	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
2335 		migrate_hrtimer_list(&old_base->clock_base[i],
2336 				     &new_base->clock_base[i]);
2337 	}
2338 
2339 	/*
2340 	 * The migration might have changed the first expiring softirq
2341 	 * timer on this CPU. Update it.
2342 	 */
2343 	__hrtimer_get_next_event(new_base, HRTIMER_ACTIVE_SOFT);
2344 	/* Tell the other CPU to retrigger the next event */
2345 	smp_call_function_single(ncpu, retrigger_next_event, NULL, 0);
2346 
2347 	raw_spin_unlock(&new_base->lock);
2348 	old_base->online = 0;
2349 	raw_spin_unlock(&old_base->lock);
2350 
2351 	return 0;
2352 }
2353 
2354 #endif /* CONFIG_HOTPLUG_CPU */
2355 
hrtimers_init(void)2356 void __init hrtimers_init(void)
2357 {
2358 	hrtimers_prepare_cpu(smp_processor_id());
2359 	hrtimers_cpu_starting(smp_processor_id());
2360 	open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
2361 }
2362