Lines Matching +full:timer +full:- +full:adjust
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(C) 2005-2006, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
4 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
5 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 * High-resolution kernel timers
9 * In contrast to the low-resolution timeout API, aka timer wheel,
16 * Based on the original timer wheel code
42 #include <linux/timer.h>
48 #include <trace/events/timer.h>
50 #include "tick-internal.h"
53 * Constants to set the queued state of the timer (INACTIVE, ENQUEUED)
56 * the timer would required touching the timer after the callback, which
57 * makes it impossible to free the timer from the callback function.
61 * timer->base->cpu_base->running == timer
64 * status. It happens for example when a posix timer expired and the callback
65 * queued a signal. Between dropping the lock which protects the posix timer
67 * signal and rearm the timer.
69 * All state transitions are protected by cpu_base->lock.
77 * idea of the (in)accuracy of timers. Timer values are rounded up to
84 * cpu_base->active
87 #define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)
95 * The timer bases:
98 * into the timer bases by the hrtimer_base_type enum. When trying
127 return likely(base->online); in hrtimer_base_is_online()
157 * timer->base->cpu_base
173 * means that all timers which are tied to this base via timer->base are
179 * When the timer's base is locked, and the timer removed from list, it is
180 * possible to set timer->base = &migration_base and drop the lock: the timer
183 static struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, in lock_hrtimer_base() argument
185 __acquires(&timer->base->lock) in lock_hrtimer_base()
188 struct hrtimer_clock_base *base = READ_ONCE(timer->base); in lock_hrtimer_base()
191 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); in lock_hrtimer_base()
192 if (likely(base == timer->base)) in lock_hrtimer_base()
194 /* The timer has migrated to another CPU: */ in lock_hrtimer_base()
195 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); in lock_hrtimer_base()
205 * If the elected target is remote and its next event is after the timer
213 * Called with cpu_base->lock of target cpu held.
215 static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base, in hrtimer_suitable_target() argument
230 * next remote target event is after this timer. Keep the in hrtimer_suitable_target()
237 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); in hrtimer_suitable_target()
239 return expires >= new_base->cpu_base->expires_next; in hrtimer_suitable_target()
258 * We switch the timer base to a power-optimized selected CPU target,
260 * - NO_HZ_COMMON is enabled
261 * - timer migration is enabled
262 * - the timer callback is not running
263 * - the timer is not the first expiring timer on the new target
265 * If one of the above requirements is not fulfilled we move the timer
267 * the timer callback is currently running.
270 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, bool pinned) in switch_hrtimer_base() argument
274 int basenum = base->index; in switch_hrtimer_base()
279 new_base = &new_cpu_base->clock_base[basenum]; in switch_hrtimer_base()
283 * We are trying to move timer to new_base. However we can't in switch_hrtimer_base()
284 * change timer's base while it is running, so we keep it on in switch_hrtimer_base()
287 * of this when the timer function has completed. There is no in switch_hrtimer_base()
288 * conflict as we hold the lock until the timer is enqueued. in switch_hrtimer_base()
290 if (unlikely(hrtimer_callback_running(timer))) in switch_hrtimer_base()
294 WRITE_ONCE(timer->base, &migration_base); in switch_hrtimer_base()
295 raw_spin_unlock(&base->cpu_base->lock); in switch_hrtimer_base()
296 raw_spin_lock(&new_base->cpu_base->lock); in switch_hrtimer_base()
298 if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) { in switch_hrtimer_base()
299 raw_spin_unlock(&new_base->cpu_base->lock); in switch_hrtimer_base()
300 raw_spin_lock(&base->cpu_base->lock); in switch_hrtimer_base()
302 WRITE_ONCE(timer->base, base); in switch_hrtimer_base()
305 WRITE_ONCE(timer->base, new_base); in switch_hrtimer_base()
307 if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) { in switch_hrtimer_base()
317 static inline struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, in lock_hrtimer_base() argument
319 __acquires(&timer->base->cpu_base->lock) in lock_hrtimer_base()
321 struct hrtimer_clock_base *base = timer->base; in lock_hrtimer_base()
323 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); in lock_hrtimer_base()
346 tmp = dclc < 0 ? -dclc : dclc; in __ktime_divns()
355 return dclc < 0 ? -tmp : tmp; in __ktime_divns()
390 * - an active object is initialized
394 struct hrtimer *timer = addr; in hrtimer_fixup_init() local
398 hrtimer_cancel(timer); in hrtimer_fixup_init()
399 debug_object_init(timer, &hrtimer_debug_descr); in hrtimer_fixup_init()
408 * - an active object is activated
409 * - an unknown non-static object is activated
424 * - an active object is freed
428 struct hrtimer *timer = addr; in hrtimer_fixup_free() local
432 hrtimer_cancel(timer); in hrtimer_fixup_free()
433 debug_object_free(timer, &hrtimer_debug_descr); in hrtimer_fixup_free()
440 /* Stub timer callback for improperly used timers. */
449 * - an untracked/uninit-ed object is found
453 struct hrtimer *timer = addr; in hrtimer_fixup_assert_init() local
457 hrtimer_setup(timer, stub_timer, CLOCK_MONOTONIC, 0); in hrtimer_fixup_assert_init()
473 static inline void debug_hrtimer_init(struct hrtimer *timer) in debug_hrtimer_init() argument
475 debug_object_init(timer, &hrtimer_debug_descr); in debug_hrtimer_init()
478 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) in debug_hrtimer_init_on_stack() argument
480 debug_object_init_on_stack(timer, &hrtimer_debug_descr); in debug_hrtimer_init_on_stack()
483 static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode) in debug_hrtimer_activate() argument
485 debug_object_activate(timer, &hrtimer_debug_descr); in debug_hrtimer_activate()
488 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) in debug_hrtimer_deactivate() argument
490 debug_object_deactivate(timer, &hrtimer_debug_descr); in debug_hrtimer_deactivate()
493 static inline void debug_hrtimer_assert_init(struct hrtimer *timer) in debug_hrtimer_assert_init() argument
495 debug_object_assert_init(timer, &hrtimer_debug_descr); in debug_hrtimer_assert_init()
498 void destroy_hrtimer_on_stack(struct hrtimer *timer) in destroy_hrtimer_on_stack() argument
500 debug_object_free(timer, &hrtimer_debug_descr); in destroy_hrtimer_on_stack()
506 static inline void debug_hrtimer_init(struct hrtimer *timer) { } in debug_hrtimer_init() argument
507 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { } in debug_hrtimer_init_on_stack() argument
508 static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode) { } in debug_hrtimer_activate() argument
509 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } in debug_hrtimer_deactivate() argument
510 static inline void debug_hrtimer_assert_init(struct hrtimer *timer) { } in debug_hrtimer_assert_init() argument
513 static inline void debug_setup(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode) in debug_setup() argument
515 debug_hrtimer_init(timer); in debug_setup()
516 trace_hrtimer_setup(timer, clockid, mode); in debug_setup()
519 static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid, in debug_setup_on_stack() argument
522 debug_hrtimer_init_on_stack(timer); in debug_setup_on_stack()
523 trace_hrtimer_setup(timer, clockid, mode); in debug_setup_on_stack()
526 static inline void debug_activate(struct hrtimer *timer, enum hrtimer_mode mode, bool was_armed) in debug_activate() argument
528 debug_hrtimer_activate(timer, mode); in debug_activate()
529 trace_hrtimer_start(timer, mode, was_armed); in debug_activate()
533 for (unsigned int idx = ffs(active); idx--; idx = ffs((active))) \
535 for (base = &cpu_base->clock_base[idx]; !done; done = true)
541 * Same as hrtimer_bases_next_event() below, but skips the excluded timer and
542 * does not update cpu_base->next_timer/expires.
551 lockdep_assert_held(&cpu_base->lock); in hrtimer_bases_next_event_without()
554 expires = ktime_sub(base->expires_next, base->offset); in hrtimer_bases_next_event_without()
559 * If the excluded timer is the first on this base evaluate the in hrtimer_bases_next_event_without()
560 * next timer. in hrtimer_bases_next_event_without()
562 struct timerqueue_linked_node *node = timerqueue_linked_first(&base->active); in hrtimer_bases_next_event_without()
564 if (unlikely(&exclude->node == node)) { in hrtimer_bases_next_event_without()
568 expires = ktime_sub(node->expires, base->offset); in hrtimer_bases_next_event_without()
574 /* If base->offset changed, the result might be negative */ in hrtimer_bases_next_event_without()
581 struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); in clock_base_next_timer()
594 expires = ktime_sub(base->expires_next, base->offset); in hrtimer_bases_first()
611 * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
618 * - HRTIMER_ACTIVE_ALL,
619 * - HRTIMER_ACTIVE_SOFT, or
620 * - HRTIMER_ACTIVE_HARD.
628 lockdep_assert_held(&cpu_base->lock); in __hrtimer_get_next_event()
630 if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) { in __hrtimer_get_next_event()
631 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; in __hrtimer_get_next_event()
634 cpu_base->softirq_next_timer = next_timer; in __hrtimer_get_next_event()
638 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; in __hrtimer_get_next_event()
641 cpu_base->next_timer = next_timer; in __hrtimer_get_next_event()
655 if (!cpu_base->softirq_activated) { in hrtimer_update_next_event()
661 cpu_base->softirq_expires_next = soft; in hrtimer_update_next_event()
666 * If a softirq timer is expiring first, update cpu_base->next_timer in hrtimer_update_next_event()
670 cpu_base->next_timer = cpu_base->softirq_next_timer; in hrtimer_update_next_event()
679 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; in hrtimer_update_base()
680 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; in hrtimer_update_base()
681 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; in hrtimer_update_base()
683 ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, offs_real, in hrtimer_update_base()
686 base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real; in hrtimer_update_base()
687 base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot; in hrtimer_update_base()
688 base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai; in hrtimer_update_base()
701 cpu_base->hres_active : 0; in hrtimer_hres_active()
713 cpu_base->expires_next = expires_next; in __hrtimer_reprogram()
719 * If a hang was detected in the last timer interrupt then we in __hrtimer_reprogram()
732 if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) in __hrtimer_reprogram()
743 if (skip_equal && expires_next == cpu_base->expires_next) in hrtimer_force_reprogram()
746 __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next); in hrtimer_force_reprogram()
749 /* High resolution timer related functions */
752 /* High resolution timer enabled ? */
764 /* hrtimer_high_res_enabled - query, if the highres mode is enabled */
776 pr_warn("Could not switch to high resolution mode on CPU %u\n", base->cpu); in hrtimer_switch_to_hres()
779 base->hres_active = true; in hrtimer_switch_to_hres()
801 * - CONFIG_HIGH_RES_TIMERS is enabled.
802 * - CONFIG_NOHZ_COMMON is enabled
817 * If high resolution mode is active then the next expiring timer in retrigger_next_event()
822 * of the next expiring timer is enough. The return from the SMP in retrigger_next_event()
829 guard(raw_spinlock)(&base->lock); in retrigger_next_event()
838 * When a timer is enqueued and expires earlier than the already enqueued
839 * timers, we have to check, whether it expires earlier than the timer for
842 * Called with interrupts disabled and base->cpu_base.lock held
844 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram) in hrtimer_reprogram() argument
847 struct hrtimer_clock_base *base = timer->base; in hrtimer_reprogram()
848 ktime_t expires = hrtimer_get_expires(timer); in hrtimer_reprogram()
852 expires = ktime_sub(expires, base->offset); in hrtimer_reprogram()
854 * CLOCK_REALTIME timer might be requested with an absolute in hrtimer_reprogram()
855 * expiry time which is less than base->offset. Set it to 0. in hrtimer_reprogram()
860 if (timer->is_soft) { in hrtimer_reprogram()
865 * first hard hrtimer on the remote CPU - in hrtimer_reprogram()
868 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base; in hrtimer_reprogram()
870 if (timer_cpu_base->softirq_activated) in hrtimer_reprogram()
873 if (!ktime_before(expires, timer_cpu_base->softirq_expires_next)) in hrtimer_reprogram()
876 timer_cpu_base->softirq_next_timer = timer; in hrtimer_reprogram()
877 timer_cpu_base->softirq_expires_next = expires; in hrtimer_reprogram()
879 if (!ktime_before(expires, timer_cpu_base->expires_next) || !reprogram) in hrtimer_reprogram()
884 * If the timer is not on the current cpu, we cannot reprogram in hrtimer_reprogram()
887 if (base->cpu_base != cpu_base) in hrtimer_reprogram()
890 if (expires >= cpu_base->expires_next) in hrtimer_reprogram()
894 if (cpu_base->deferred_rearm) in hrtimer_reprogram()
897 cpu_base->next_timer = timer; in hrtimer_reprogram()
899 __hrtimer_reprogram(cpu_base, timer, expires); in hrtimer_reprogram()
916 * the next expiring timer. in update_needs_ipi()
918 seq = cpu_base->clock_was_set_seq; in update_needs_ipi()
925 if (seq == cpu_base->clock_was_set_seq) in update_needs_ipi()
929 if (cpu_base->deferred_rearm) { in update_needs_ipi()
930 cpu_base->deferred_needs_update = true; in update_needs_ipi()
936 * timer in a clock base is moving ahead of the first expiring timer of in update_needs_ipi()
940 active &= cpu_base->active_bases; in update_needs_ipi()
945 next = timerqueue_linked_first(&base->active); in update_needs_ipi()
946 expires = ktime_sub(next->expires, base->offset); in update_needs_ipi()
947 if (expires < cpu_base->expires_next) in update_needs_ipi()
951 if (base->index < HRTIMER_BASE_MONOTONIC_SOFT) in update_needs_ipi()
953 if (cpu_base->softirq_activated) in update_needs_ipi()
955 if (expires < cpu_base->softirq_expires_next) in update_needs_ipi()
968 * when the change moves an affected timer ahead of the first expiring
969 * timer on that CPU. Obviously remote per CPU clock event devices cannot
994 guard(raw_spinlock_irqsave)(&cpu_base->lock); in clock_was_set()
1036 static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) in unlock_hrtimer_base() argument
1037 __releases(&timer->base->cpu_base->lock) in unlock_hrtimer_base()
1039 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); in unlock_hrtimer_base()
1043 * hrtimer_forward() - forward the timer expiry
1044 * @timer: hrtimer to forward
1048 * Forward the timer expiry so it will expire in the future.
1051 * This only updates the timer expiry value and does not requeue the timer.
1055 * Context: Can be safely called from the callback function of @timer. If called
1056 * from other contexts @timer must neither be enqueued nor running the
1061 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) in hrtimer_forward() argument
1066 delta = ktime_sub(now, hrtimer_get_expires(timer)); in hrtimer_forward()
1071 if (WARN_ON(timer->is_queued)) in hrtimer_forward()
1081 hrtimer_add_expires_ns(timer, incr * orun); in hrtimer_forward()
1082 if (hrtimer_get_expires(timer) > now) in hrtimer_forward()
1090 hrtimer_add_expires(timer, interval); in hrtimer_forward()
1097 * enqueue_hrtimer - internal function to (re)start a timer
1099 * The timer is inserted in expiry order. Insertion into the
1102 * Returns true when the new timer is the leftmost timer in the tree.
1104 static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, in enqueue_hrtimer() argument
1107 lockdep_assert_held(&base->cpu_base->lock); in enqueue_hrtimer()
1109 debug_activate(timer, mode, was_armed); in enqueue_hrtimer()
1110 WARN_ON_ONCE(!base->cpu_base->online); in enqueue_hrtimer()
1112 base->cpu_base->active_bases |= 1 << base->index; in enqueue_hrtimer()
1115 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED); in enqueue_hrtimer()
1117 if (!timerqueue_linked_add(&base->active, &timer->node)) in enqueue_hrtimer()
1120 base->expires_next = hrtimer_get_expires(timer); in enqueue_hrtimer()
1126 struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); in base_update_next_timer()
1128 base->expires_next = next ? next->expires : KTIME_MAX; in base_update_next_timer()
1132 * __remove_hrtimer - internal function to remove a timer
1134 * High resolution timer mode reprograms the clock event device when the
1135 * timer is the one which expires next. The caller can disable this by setting
1137 * anyway (e.g. timer interrupt)
1139 static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, in __remove_hrtimer() argument
1142 struct hrtimer_cpu_base *cpu_base = base->cpu_base; in __remove_hrtimer()
1145 lockdep_assert_held(&cpu_base->lock); in __remove_hrtimer()
1147 if (!timer->is_queued) in __remove_hrtimer()
1151 WRITE_ONCE(timer->is_queued, newstate); in __remove_hrtimer()
1153 was_first = !timerqueue_linked_prev(&timer->node); in __remove_hrtimer()
1155 if (!timerqueue_linked_del(&base->active, &timer->node)) in __remove_hrtimer()
1156 cpu_base->active_bases &= ~(1 << base->index); in __remove_hrtimer()
1158 /* Nothing to update if this was not the first timer in the base */ in __remove_hrtimer()
1165 * If reprogram is false don't update cpu_base->next_timer and do not in __remove_hrtimer()
1168 * This happens when removing the first timer on a remote CPU, which in __remove_hrtimer()
1170 * a local timer is removed to be immediately restarted. That's handled in __remove_hrtimer()
1173 if (!reprogram || timer != cpu_base->next_timer || timer->is_lazy) in __remove_hrtimer()
1176 if (cpu_base->deferred_rearm) in __remove_hrtimer()
1177 cpu_base->deferred_needs_update = true; in __remove_hrtimer()
1182 static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, in remove_hrtimer() argument
1185 lockdep_assert_held(&base->cpu_base->lock); in remove_hrtimer()
1187 if (timer->is_queued) { in remove_hrtimer()
1190 debug_hrtimer_deactivate(timer); in remove_hrtimer()
1193 * Remove the timer and force reprogramming when high in remove_hrtimer()
1194 * resolution mode is active and the timer is on the current in remove_hrtimer()
1195 * CPU. If we remove a timer on another CPU, reprogramming is in remove_hrtimer()
1200 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); in remove_hrtimer()
1202 __remove_hrtimer(timer, base, newstate, reprogram); in remove_hrtimer()
1216 hrtimer_can_update_in_place(struct hrtimer *timer, struct hrtimer_clock_base *base, ktime_t expires) in hrtimer_can_update_in_place() argument
1218 struct timerqueue_linked_node *next = timerqueue_linked_next(&timer->node); in hrtimer_can_update_in_place()
1219 struct timerqueue_linked_node *prev = timerqueue_linked_prev(&timer->node); in hrtimer_can_update_in_place()
1221 /* If the new expiry goes behind the next timer, requeue is required */ in hrtimer_can_update_in_place()
1222 if (next && expires > next->expires) in hrtimer_can_update_in_place()
1225 /* If this is the first timer, update in place */ in hrtimer_can_update_in_place()
1230 return expires >= prev->expires; in hrtimer_can_update_in_place()
1234 remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base, in remove_and_enqueue_same_base() argument
1239 /* Remove it from the timer queue if active */ in remove_and_enqueue_same_base()
1240 if (timer->is_queued) { in remove_and_enqueue_same_base()
1241 was_first = !timerqueue_linked_prev(&timer->node); in remove_and_enqueue_same_base()
1244 if (hrtimer_can_update_in_place(timer, base, expires)) { in remove_and_enqueue_same_base()
1245 hrtimer_set_expires_range_ns(timer, expires, delta_ns); in remove_and_enqueue_same_base()
1246 trace_hrtimer_start(timer, mode, true); in remove_and_enqueue_same_base()
1248 base->expires_next = expires; in remove_and_enqueue_same_base()
1252 debug_hrtimer_deactivate(timer); in remove_and_enqueue_same_base()
1253 timerqueue_linked_del(&base->active, &timer->node); in remove_and_enqueue_same_base()
1257 hrtimer_set_expires_range_ns(timer, expires, delta_ns); in remove_and_enqueue_same_base()
1259 debug_activate(timer, mode, timer->is_queued); in remove_and_enqueue_same_base()
1260 base->cpu_base->active_bases |= 1 << base->index; in remove_and_enqueue_same_base()
1263 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED); in remove_and_enqueue_same_base()
1265 /* If it's the first expiring timer now or again, update base */ in remove_and_enqueue_same_base()
1266 if (timerqueue_linked_add(&base->active, &timer->node)) { in remove_and_enqueue_same_base()
1267 base->expires_next = expires; in remove_and_enqueue_same_base()
1277 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim, in hrtimer_update_lowres() argument
1286 timer->is_rel = mode & HRTIMER_MODE_REL; in hrtimer_update_lowres()
1287 if (timer->is_rel) in hrtimer_update_lowres()
1300 * hrtimer. cpu_base->softirq_expires_next needs to be updated! in hrtimer_update_softirq_timer()
1306 * cpu_base->next_timer is recomputed by __hrtimer_get_next_event() in hrtimer_update_softirq_timer()
1307 * cpu_base->expires_next is only set by hrtimer_reprogram() in hrtimer_update_softirq_timer()
1309 hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram); in hrtimer_update_softirq_timer()
1317 * If it is local and the first expiring timer keep it on the local in hrtimer_prefer_local()
1332 * there is no point in moving the timer somewhere else. in hrtimer_prefer_local()
1345 static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool is_first, in hrtimer_keep_base() argument
1348 /* If the timer is running the callback it has to stay on its CPU base. */ in hrtimer_keep_base()
1349 if (unlikely(timer->base->running == timer)) in hrtimer_keep_base()
1355 static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, in __hrtimer_start_range_ns() argument
1360 struct hrtimer_cpu_base *cpu_base = base->cpu_base; in __hrtimer_start_range_ns()
1362 was_first = cpu_base->next_timer == timer; in __hrtimer_start_range_ns()
1369 if (likely(this_cpu_base->online)) { in __hrtimer_start_range_ns()
1372 keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned); in __hrtimer_start_range_ns()
1377 tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid)); in __hrtimer_start_range_ns()
1379 tim = hrtimer_update_lowres(timer, tim, mode); in __hrtimer_start_range_ns()
1382 * Remove an active timer from the queue. In case it is not queued in __hrtimer_start_range_ns()
1386 * If it's on the current CPU and the first expiring timer, then in __hrtimer_start_range_ns()
1387 * skip reprogramming, keep the timer local and enforce in __hrtimer_start_range_ns()
1388 * reprogramming later if it was the first expiring timer. This in __hrtimer_start_range_ns()
1392 * @keep_base is also true if the timer callback is running on a in __hrtimer_start_range_ns()
1396 first = remove_and_enqueue_same_base(timer, base, mode, tim, delta_ns); in __hrtimer_start_range_ns()
1399 bool was_armed = remove_hrtimer(timer, base, HRTIMER_STATE_ENQUEUED); in __hrtimer_start_range_ns()
1401 hrtimer_set_expires_range_ns(timer, tim, delta_ns); in __hrtimer_start_range_ns()
1403 /* Switch the timer base, if necessary: */ in __hrtimer_start_range_ns()
1404 base = switch_hrtimer_base(timer, base, is_pinned); in __hrtimer_start_range_ns()
1405 cpu_base = base->cpu_base; in __hrtimer_start_range_ns()
1407 first = enqueue_hrtimer(timer, base, mode, was_armed); in __hrtimer_start_range_ns()
1411 if (cpu_base->deferred_rearm) { in __hrtimer_start_range_ns()
1412 cpu_base->deferred_needs_update = true; in __hrtimer_start_range_ns()
1418 * If the current CPU base is online, then the timer is never in __hrtimer_start_range_ns()
1420 * timer there unless the timer callback is currently executed in __hrtimer_start_range_ns()
1422 * re-evaluate the first expiring timer after completing the in __hrtimer_start_range_ns()
1429 * Timer was enqueued remote because the current base is in __hrtimer_start_range_ns()
1430 * already offline. If the timer is the first to expire, in __hrtimer_start_range_ns()
1434 smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd); in __hrtimer_start_range_ns()
1439 * Special case for the HRTICK timer. It is frequently rearmed and most in __hrtimer_start_range_ns()
1447 if (timer->is_lazy) { in __hrtimer_start_range_ns()
1448 if (cpu_base->expires_next <= hrtimer_get_expires(timer)) in __hrtimer_start_range_ns()
1453 * Timer was the first expiring timer and forced to stay on the in __hrtimer_start_range_ns()
1456 * timer. in __hrtimer_start_range_ns()
1463 * hrtimer_start_range_ns - (re)start an hrtimer
1464 * @timer: the timer to be added
1466 * @delta_ns: "slack" range for the timer
1467 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
1471 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, in hrtimer_start_range_ns() argument
1477 debug_hrtimer_assert_init(timer); in hrtimer_start_range_ns()
1485 WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); in hrtimer_start_range_ns()
1487 WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard); in hrtimer_start_range_ns()
1489 base = lock_hrtimer_base(timer, &flags); in hrtimer_start_range_ns()
1491 if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base)) in hrtimer_start_range_ns()
1492 hrtimer_reprogram(timer, true); in hrtimer_start_range_ns()
1494 unlock_hrtimer_base(timer, &flags); in hrtimer_start_range_ns()
1499 * hrtimer_try_to_cancel - try to deactivate a timer
1500 * @timer: hrtimer to stop
1504 * * 0 when the timer was not active
1505 * * 1 when the timer was active
1506 * * -1 when the timer is currently executing the callback function and
1509 int hrtimer_try_to_cancel(struct hrtimer *timer) in hrtimer_try_to_cancel() argument
1513 int ret = -1; in hrtimer_try_to_cancel()
1516 * Check lockless first. If the timer is not active (neither in hrtimer_try_to_cancel()
1521 if (!hrtimer_active(timer)) in hrtimer_try_to_cancel()
1524 base = lock_hrtimer_base(timer, &flags); in hrtimer_try_to_cancel()
1526 if (!hrtimer_callback_running(timer)) { in hrtimer_try_to_cancel()
1527 ret = remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE); in hrtimer_try_to_cancel()
1529 trace_hrtimer_cancel(timer); in hrtimer_try_to_cancel()
1532 unlock_hrtimer_base(timer, &flags); in hrtimer_try_to_cancel()
1542 spin_lock_init(&base->softirq_expiry_lock); in hrtimer_cpu_base_init_expiry_lock()
1546 __acquires(&base->softirq_expiry_lock) in hrtimer_cpu_base_lock_expiry()
1548 spin_lock(&base->softirq_expiry_lock); in hrtimer_cpu_base_lock_expiry()
1552 __releases(&base->softirq_expiry_lock) in hrtimer_cpu_base_unlock_expiry()
1554 spin_unlock(&base->softirq_expiry_lock); in hrtimer_cpu_base_unlock_expiry()
1560 * If there is a waiter for cpu_base->expiry_lock, then it was waiting for
1561 * the timer callback to finish. Drop expiry_lock and reacquire it. That
1566 if (atomic_read(&cpu_base->timer_waiters)) { in hrtimer_sync_wait_running()
1567 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_sync_wait_running()
1568 spin_unlock(&cpu_base->softirq_expiry_lock); in hrtimer_sync_wait_running()
1569 spin_lock(&cpu_base->softirq_expiry_lock); in hrtimer_sync_wait_running()
1570 raw_spin_lock_irq(&cpu_base->lock); in hrtimer_sync_wait_running()
1588 * deletion of a timer failed because the timer callback function was
1592 * in the middle of a timer callback, then calling hrtimer_cancel() can
1595 * - If the caller is on a remote CPU then it has to spin wait for the timer
1598 * - If the caller originates from the task which preempted the timer
1599 * handler on the same CPU, then spin waiting for the timer handler to
1602 void hrtimer_cancel_wait_running(const struct hrtimer *timer) in hrtimer_cancel_wait_running() argument
1605 struct hrtimer_clock_base *base = READ_ONCE(timer->base); in hrtimer_cancel_wait_running()
1608 * Just relax if the timer expires in hard interrupt context or if in hrtimer_cancel_wait_running()
1611 if (!timer->is_soft || is_migration_base(base)) { in hrtimer_cancel_wait_running()
1618 * held by the softirq across the timer callback. Drop the lock in hrtimer_cancel_wait_running()
1619 * immediately so the softirq can expire the next timer. In theory in hrtimer_cancel_wait_running()
1620 * the timer could already be running again, but that's more than in hrtimer_cancel_wait_running()
1623 atomic_inc(&base->cpu_base->timer_waiters); in hrtimer_cancel_wait_running()
1624 spin_lock_bh(&base->cpu_base->softirq_expiry_lock); in hrtimer_cancel_wait_running()
1625 atomic_dec(&base->cpu_base->timer_waiters); in hrtimer_cancel_wait_running()
1626 spin_unlock_bh(&base->cpu_base->softirq_expiry_lock); in hrtimer_cancel_wait_running()
1636 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
1637 * @timer: the timer to be cancelled
1640 * 0 when the timer was not active
1641 * 1 when the timer was active
1643 int hrtimer_cancel(struct hrtimer *timer) in hrtimer_cancel() argument
1648 ret = hrtimer_try_to_cancel(timer); in hrtimer_cancel()
1651 hrtimer_cancel_wait_running(timer); in hrtimer_cancel()
1658 * __hrtimer_get_remaining - get remaining time for the timer
1659 * @timer: the timer to read
1660 * @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y
1662 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust) in __hrtimer_get_remaining() argument
1667 lock_hrtimer_base(timer, &flags); in __hrtimer_get_remaining()
1668 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust) in __hrtimer_get_remaining()
1669 rem = hrtimer_expires_remaining_adjusted(timer); in __hrtimer_get_remaining()
1671 rem = hrtimer_expires_remaining(timer); in __hrtimer_get_remaining()
1672 unlock_hrtimer_base(timer, &flags); in __hrtimer_get_remaining()
1680 * hrtimer_get_next_event - get the time until next expiry event
1682 * Returns the next expiry time or KTIME_MAX if no timer is pending.
1689 guard(raw_spinlock_irqsave)(&cpu_base->lock); in hrtimer_get_next_event()
1697 * hrtimer_next_event_without - time until next expiry event w/o one timer
1698 * @exclude: timer to exclude
1709 guard(raw_spinlock_irqsave)(&cpu_base->lock); in hrtimer_next_event_without()
1713 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; in hrtimer_next_event_without()
1714 if (active && !cpu_base->softirq_activated) in hrtimer_next_event_without()
1717 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; in hrtimer_next_event_without()
1758 ktime_t hrtimer_cb_get_time(const struct hrtimer *timer) in hrtimer_cb_get_time() argument
1760 return __hrtimer_cb_get_time(timer->base->clockid); in hrtimer_cb_get_time()
1764 static void __hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*fn)(struct hrtimer *), in __hrtimer_setup() argument
1780 memset(timer, 0, sizeof(struct hrtimer)); in __hrtimer_setup()
1794 timer->is_soft = softtimer; in __hrtimer_setup()
1795 timer->is_hard = !!(mode & HRTIMER_MODE_HARD); in __hrtimer_setup()
1796 timer->is_lazy = !!(mode & HRTIMER_MODE_LAZY_REARM); in __hrtimer_setup()
1797 timer->base = &cpu_base->clock_base[base]; in __hrtimer_setup()
1798 timerqueue_linked_init(&timer->node); in __hrtimer_setup()
1801 ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout; in __hrtimer_setup()
1803 ACCESS_PRIVATE(timer, function) = fn; in __hrtimer_setup()
1807 * hrtimer_setup - initialize a timer to the given clock
1808 * @timer: the timer to be initialized
1819 void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *), in hrtimer_setup() argument
1822 debug_setup(timer, clock_id, mode); in hrtimer_setup()
1823 __hrtimer_setup(timer, function, clock_id, mode); in hrtimer_setup()
1828 * hrtimer_setup_on_stack - initialize a timer on stack memory
1829 * @timer: The timer to be initialized
1832 * @mode: The timer mode
1837 void hrtimer_setup_on_stack(struct hrtimer *timer, in hrtimer_setup_on_stack() argument
1841 debug_setup_on_stack(timer, clock_id, mode); in hrtimer_setup_on_stack()
1842 __hrtimer_setup(timer, function, clock_id, mode); in hrtimer_setup_on_stack()
1847 * A timer is active, when it is enqueued into the rbtree or the
1853 bool hrtimer_active(const struct hrtimer *timer) in hrtimer_active() argument
1859 base = READ_ONCE(timer->base); in hrtimer_active()
1860 seq = raw_read_seqcount_begin(&base->seq); in hrtimer_active()
1862 if (timer->is_queued || base->running == timer) in hrtimer_active()
1865 } while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base)); in hrtimer_active()
1875 * - queued: the timer is queued
1876 * - callback: the timer is being ran
1877 * - post: the timer is inactive or (re)queued
1879 * On the read side we ensure we observe timer->is_queued and cpu_base->running
1881 * This includes timer->base changing because sequence numbers alone are
1889 struct hrtimer *timer, ktime_t now, unsigned long flags) in __run_hrtimer() argument
1890 __must_hold(&cpu_base->lock) in __run_hrtimer()
1896 lockdep_assert_held(&cpu_base->lock); in __run_hrtimer()
1898 debug_hrtimer_deactivate(timer); in __run_hrtimer()
1899 base->running = timer; in __run_hrtimer()
1902 * Separate the ->running assignment from the ->is_queued assignment. in __run_hrtimer()
1905 * hrtimer_active() cannot observe base->running == NULL && in __run_hrtimer()
1906 * timer->is_queued == INACTIVE. in __run_hrtimer()
1908 raw_write_seqcount_barrier(&base->seq); in __run_hrtimer()
1910 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, false); in __run_hrtimer()
1911 fn = ACCESS_PRIVATE(timer, function); in __run_hrtimer()
1915 * timer is restarted with a period then it becomes an absolute in __run_hrtimer()
1916 * timer. If its not restarted it does not matter. in __run_hrtimer()
1919 timer->is_rel = false; in __run_hrtimer()
1922 * The timer is marked as running in the CPU base, so it is in __run_hrtimer()
1926 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in __run_hrtimer()
1927 trace_hrtimer_expire_entry(timer, now); in __run_hrtimer()
1928 expires_in_hardirq = lockdep_hrtimer_enter(timer); in __run_hrtimer()
1930 restart = fn(timer); in __run_hrtimer()
1933 trace_hrtimer_expire_exit(timer); in __run_hrtimer()
1934 raw_spin_lock_irq(&cpu_base->lock); in __run_hrtimer()
1941 * Note: Because we dropped the cpu_base->lock above, in __run_hrtimer()
1942 * hrtimer_start_range_ns() can have popped in and enqueued the timer in __run_hrtimer()
1945 if (restart == HRTIMER_RESTART && !timer->is_queued) in __run_hrtimer()
1946 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS, false); in __run_hrtimer()
1949 * Separate the ->running assignment from the ->is_queued assignment. in __run_hrtimer()
1952 * hrtimer_active() cannot observe base->running.timer == NULL && in __run_hrtimer()
1953 * timer->is_queued == INACTIVE. in __run_hrtimer()
1955 raw_write_seqcount_barrier(&base->seq); in __run_hrtimer()
1957 WARN_ON_ONCE(base->running != timer); in __run_hrtimer()
1958 base->running = NULL; in __run_hrtimer()
1963 struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); in clock_base_next_timer_safe()
1971 unsigned int active = cpu_base->active_bases & active_mask; in __hrtimer_run_queues()
1975 ktime_t basenow = ktime_add(now, base->offset); in __hrtimer_run_queues()
1976 struct hrtimer *timer; in __hrtimer_run_queues() local
1978 while ((timer = clock_base_next_timer(base))) { in __hrtimer_run_queues()
1988 * are right-of a not yet expired timer, because that in __hrtimer_run_queues()
1989 * timer will have to trigger a wakeup anyway. in __hrtimer_run_queues()
1991 if (basenow < hrtimer_get_softexpires(timer)) in __hrtimer_run_queues()
1994 __run_hrtimer(cpu_base, base, timer, basenow, flags); in __hrtimer_run_queues()
2008 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_run_softirq()
2013 cpu_base->softirq_activated = false; in hrtimer_run_softirq()
2016 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_run_softirq()
2028 cpu_base->expires_next = expires_next; in hrtimer_rearm()
2029 cpu_base->deferred_rearm = false; in hrtimer_rearm()
2031 if (unlikely(cpu_base->hang_detected)) { in hrtimer_rearm()
2037 min(100 * NSEC_PER_MSEC, cpu_base->max_hang_time)); in hrtimer_rearm()
2048 if (!cpu_base->deferred_rearm) in __hrtimer_rearm_deferred()
2051 guard(raw_spinlock)(&cpu_base->lock); in __hrtimer_rearm_deferred()
2052 if (cpu_base->deferred_needs_update) { in __hrtimer_rearm_deferred()
2056 /* No timer added/removed. Use the cached value */ in __hrtimer_rearm_deferred()
2057 expires_next = cpu_base->deferred_expires_next; in __hrtimer_rearm_deferred()
2065 /* hrtimer_interrupt() just re-evaluated the first expiring timer */ in hrtimer_interrupt_rearm()
2066 cpu_base->deferred_needs_update = false; in hrtimer_interrupt_rearm()
2068 cpu_base->deferred_expires_next = expires_next; in hrtimer_interrupt_rearm()
2080 * High resolution timer interrupt
2090 BUG_ON(!cpu_base->hres_active); in hrtimer_interrupt()
2091 cpu_base->nr_events++; in hrtimer_interrupt()
2092 dev->next_event = KTIME_MAX; in hrtimer_interrupt()
2093 dev->next_event_forced = 0; in hrtimer_interrupt()
2095 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_interrupt()
2098 cpu_base->deferred_rearm = true; in hrtimer_interrupt()
2105 cpu_base->expires_next = KTIME_MAX; in hrtimer_interrupt()
2107 if (!ktime_before(now, cpu_base->softirq_expires_next)) { in hrtimer_interrupt()
2108 cpu_base->softirq_expires_next = KTIME_MAX; in hrtimer_interrupt()
2109 cpu_base->softirq_activated = true; in hrtimer_interrupt()
2116 * The next timer was already expired due to: in hrtimer_interrupt()
2117 * - tracing in hrtimer_interrupt()
2118 * - long lasting callbacks in hrtimer_interrupt()
2119 * - being scheduled away when running in a VM in hrtimer_interrupt()
2127 cpu_base->hang_detected = false; in hrtimer_interrupt()
2133 cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta); in hrtimer_interrupt()
2134 cpu_base->nr_hangs++; in hrtimer_interrupt()
2135 cpu_base->hang_detected = true; in hrtimer_interrupt()
2139 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_interrupt()
2168 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_run_queues()
2171 if (!ktime_before(now, cpu_base->softirq_expires_next)) { in hrtimer_run_queues()
2172 cpu_base->softirq_expires_next = KTIME_MAX; in hrtimer_run_queues()
2173 cpu_base->softirq_activated = true; in hrtimer_run_queues()
2178 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_run_queues()
2184 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) in hrtimer_wakeup() argument
2186 struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); in hrtimer_wakeup()
2187 struct task_struct *task = t->task; in hrtimer_wakeup()
2189 t->task = NULL; in hrtimer_wakeup()
2197 * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
2199 * @mode: timer mode abs/rel
2213 if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard) in hrtimer_sleeper_start_expires()
2216 hrtimer_start_expires(&sl->timer, mode); in hrtimer_sleeper_start_expires()
2237 * OTOH, privileged real-time user space applications rely on the in __hrtimer_setup_sleeper()
2239 * a real-time scheduling class, mark the mode for hard interrupt in __hrtimer_setup_sleeper()
2247 __hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode); in __hrtimer_setup_sleeper()
2248 sl->task = current; in __hrtimer_setup_sleeper()
2252 * hrtimer_setup_sleeper_on_stack - initialize a sleeper in stack memory
2255 * @mode: timer mode abs/rel
2260 debug_setup_on_stack(&sl->timer, clock_id, mode); in hrtimer_setup_sleeper_on_stack()
2267 switch(restart->nanosleep.type) { in nanosleep_copyout()
2270 if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp)) in nanosleep_copyout()
2271 return -EFAULT; in nanosleep_copyout()
2275 if (put_timespec64(ts, restart->nanosleep.rmtp)) in nanosleep_copyout()
2276 return -EFAULT; in nanosleep_copyout()
2281 return -ERESTART_RESTARTBLOCK; in nanosleep_copyout()
2292 if (likely(t->task)) in do_nanosleep()
2295 hrtimer_cancel(&t->timer); in do_nanosleep()
2298 } while (t->task && !signal_pending(current)); in do_nanosleep()
2302 if (!t->task) in do_nanosleep()
2305 restart = ¤t->restart_block; in do_nanosleep()
2306 if (restart->nanosleep.type != TT_NONE) { in do_nanosleep()
2307 ktime_t rem = hrtimer_expires_remaining(&t->timer); in do_nanosleep()
2316 return -ERESTART_RESTARTBLOCK; in do_nanosleep()
2324 hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS); in hrtimer_nanosleep_restart()
2325 hrtimer_set_expires(&t.timer, restart->nanosleep.expires); in hrtimer_nanosleep_restart()
2327 destroy_hrtimer_on_stack(&t.timer); in hrtimer_nanosleep_restart()
2338 hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns); in hrtimer_nanosleep()
2340 if (ret != -ERESTART_RESTARTBLOCK) in hrtimer_nanosleep()
2345 ret = -ERESTARTNOHAND; in hrtimer_nanosleep()
2349 restart = ¤t->restart_block; in hrtimer_nanosleep()
2350 restart->nanosleep.clockid = t.timer.base->clockid; in hrtimer_nanosleep()
2351 restart->nanosleep.expires = hrtimer_get_expires(&t.timer); in hrtimer_nanosleep()
2354 destroy_hrtimer_on_stack(&t.timer); in hrtimer_nanosleep()
2366 return -EFAULT; in SYSCALL_DEFINE2()
2369 return -EINVAL; in SYSCALL_DEFINE2()
2371 current->restart_block.fn = do_no_restart_syscall; in SYSCALL_DEFINE2()
2372 current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE; in SYSCALL_DEFINE2()
2373 current->restart_block.nanosleep.rmtp = rmtp; in SYSCALL_DEFINE2()
2387 return -EFAULT; in SYSCALL_DEFINE2()
2390 return -EINVAL; in SYSCALL_DEFINE2()
2392 current->restart_block.fn = do_no_restart_syscall; in SYSCALL_DEFINE2()
2393 current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE; in SYSCALL_DEFINE2()
2394 current->restart_block.nanosleep.compat_rmtp = rmtp; in SYSCALL_DEFINE2()
2400 * Functions related to boot-time initialization:
2407 struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i]; in hrtimers_prepare_cpu()
2409 clock_b->cpu_base = cpu_base; in hrtimers_prepare_cpu()
2410 seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock); in hrtimers_prepare_cpu()
2411 timerqueue_linked_init_head(&clock_b->active); in hrtimers_prepare_cpu()
2414 cpu_base->cpu = cpu; in hrtimers_prepare_cpu()
2424 cpu_base->active_bases = 0; in hrtimers_cpu_starting()
2425 cpu_base->hres_active = false; in hrtimers_cpu_starting()
2426 cpu_base->hang_detected = false; in hrtimers_cpu_starting()
2427 cpu_base->next_timer = NULL; in hrtimers_cpu_starting()
2428 cpu_base->softirq_next_timer = NULL; in hrtimers_cpu_starting()
2429 cpu_base->expires_next = KTIME_MAX; in hrtimers_cpu_starting()
2430 cpu_base->softirq_expires_next = KTIME_MAX; in hrtimers_cpu_starting()
2431 cpu_base->softirq_activated = false; in hrtimers_cpu_starting()
2432 cpu_base->online = true; in hrtimers_cpu_starting()
2442 struct hrtimer *timer; in migrate_hrtimer_list() local
2444 while ((node = timerqueue_linked_first(&old_base->active))) { in migrate_hrtimer_list()
2445 timer = hrtimer_from_timerqueue_node(node); in migrate_hrtimer_list()
2446 BUG_ON(hrtimer_callback_running(timer)); in migrate_hrtimer_list()
2447 debug_hrtimer_deactivate(timer); in migrate_hrtimer_list()
2451 * timer could be seen as !active and just vanish away in migrate_hrtimer_list()
2454 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, false); in migrate_hrtimer_list()
2455 timer->base = new_base; in migrate_hrtimer_list()
2458 * reprogram the event device in case the timer in migrate_hrtimer_list()
2464 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS, true); in migrate_hrtimer_list()
2480 raw_spin_lock(&old_base->lock); in hrtimers_cpu_dying()
2481 raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING); in hrtimers_cpu_dying()
2484 migrate_hrtimer_list(&old_base->clock_base[i], &new_base->clock_base[i]); in hrtimers_cpu_dying()
2489 raw_spin_unlock(&new_base->lock); in hrtimers_cpu_dying()
2490 old_base->online = false; in hrtimers_cpu_dying()
2491 raw_spin_unlock(&old_base->lock); in hrtimers_cpu_dying()