163 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,  in lock_hrtimer_base()  argument
165 	__acquires(&timer->base->lock)  in lock_hrtimer_base()
170 		base = READ_ONCE(timer->base);  in lock_hrtimer_base()
173 			if (likely(base == timer->base))  in lock_hrtimer_base()
196 static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base,  in hrtimer_suitable_target()  argument
218 	expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);  in hrtimer_suitable_target()
251 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,  in switch_hrtimer_base()  argument
273 		if (unlikely(hrtimer_callback_running(timer)))  in switch_hrtimer_base()
277 		WRITE_ONCE(timer->base, &migration_base);  in switch_hrtimer_base()
281 		if (!hrtimer_suitable_target(timer, new_base, new_cpu_base,  in switch_hrtimer_base()
286 			WRITE_ONCE(timer->base, base);  in switch_hrtimer_base()
289 		WRITE_ONCE(timer->base, new_base);  in switch_hrtimer_base()
291 		if (!hrtimer_suitable_target(timer, new_base,  new_cpu_base, this_cpu_base)) {  in switch_hrtimer_base()
302 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in lock_hrtimer_base()  argument
303 	__acquires(&timer->base->cpu_base->lock)  in lock_hrtimer_base()
305 	struct hrtimer_clock_base *base = timer->base;  in lock_hrtimer_base()
379 	struct hrtimer *timer = addr;  in hrtimer_fixup_init()  local
383 		hrtimer_cancel(timer);  in hrtimer_fixup_init()
384 		debug_object_init(timer, &hrtimer_debug_descr);  in hrtimer_fixup_init()
413 	struct hrtimer *timer = addr;  in hrtimer_fixup_free()  local
417 		hrtimer_cancel(timer);  in hrtimer_fixup_free()
418 		debug_object_free(timer, &hrtimer_debug_descr);  in hrtimer_fixup_free()
433 static inline void debug_hrtimer_init(struct hrtimer *timer)  in debug_hrtimer_init()  argument
435 	debug_object_init(timer, &hrtimer_debug_descr);  in debug_hrtimer_init()
438 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer)  in debug_hrtimer_init_on_stack()  argument
440 	debug_object_init_on_stack(timer, &hrtimer_debug_descr);  in debug_hrtimer_init_on_stack()
443 static inline void debug_hrtimer_activate(struct hrtimer *timer,  in debug_hrtimer_activate()  argument
446 	debug_object_activate(timer, &hrtimer_debug_descr);  in debug_hrtimer_activate()
449 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)  in debug_hrtimer_deactivate()  argument
451 	debug_object_deactivate(timer, &hrtimer_debug_descr);  in debug_hrtimer_deactivate()
454 void destroy_hrtimer_on_stack(struct hrtimer *timer)  in destroy_hrtimer_on_stack()  argument
456 	debug_object_free(timer, &hrtimer_debug_descr);  in destroy_hrtimer_on_stack()
462 static inline void debug_hrtimer_init(struct hrtimer *timer) { }  in debug_hrtimer_init()  argument
463 static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }  in debug_hrtimer_init_on_stack()  argument
464 static inline void debug_hrtimer_activate(struct hrtimer *timer,  in debug_hrtimer_activate()  argument
466 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }  in debug_hrtimer_deactivate()  argument
469 static inline void debug_setup(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode)  in debug_setup()  argument
471 	debug_hrtimer_init(timer);  in debug_setup()
472 	trace_hrtimer_setup(timer, clockid, mode);  in debug_setup()
475 static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid,  in debug_setup_on_stack()  argument
478 	debug_hrtimer_init_on_stack(timer);  in debug_setup_on_stack()
479 	trace_hrtimer_setup(timer, clockid, mode);  in debug_setup_on_stack()
482 static inline void debug_activate(struct hrtimer *timer,  in debug_activate()  argument
485 	debug_hrtimer_activate(timer, mode);  in debug_activate()
486 	trace_hrtimer_start(timer, mode);  in debug_activate()
489 static inline void debug_deactivate(struct hrtimer *timer)  in debug_deactivate()  argument
491 	debug_hrtimer_deactivate(timer);  in debug_deactivate()
492 	trace_hrtimer_cancel(timer);  in debug_deactivate()
522 		struct hrtimer *timer;  in __hrtimer_next_event_base()  local
525 		timer = container_of(next, struct hrtimer, node);  in __hrtimer_next_event_base()
526 		if (timer == exclude) {  in __hrtimer_next_event_base()
532 			timer = container_of(next, struct hrtimer, node);  in __hrtimer_next_event_base()
534 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in __hrtimer_next_event_base()
542 			if (timer->is_soft)  in __hrtimer_next_event_base()
543 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
545 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
811 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)  in hrtimer_reprogram()  argument
814 	struct hrtimer_clock_base *base = timer->base;  in hrtimer_reprogram()
815 	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in hrtimer_reprogram()
817 	WARN_ON_ONCE(hrtimer_get_expires(timer) < 0);  in hrtimer_reprogram()
826 	if (timer->is_soft) {  in hrtimer_reprogram()
842 		timer_cpu_base->softirq_next_timer = timer;  in hrtimer_reprogram()
867 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
869 	__hrtimer_reprogram(cpu_base, timer, expires);  in hrtimer_reprogram()
1018 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in unlock_hrtimer_base()  argument
1019 	__releases(&timer->base->cpu_base->lock)  in unlock_hrtimer_base()
1021 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
1043 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)  in hrtimer_forward()  argument
1048 	delta = ktime_sub(now, hrtimer_get_expires(timer));  in hrtimer_forward()
1053 	if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))  in hrtimer_forward()
1063 		hrtimer_add_expires_ns(timer, incr * orun);  in hrtimer_forward()
1064 		if (hrtimer_get_expires(timer) > now)  in hrtimer_forward()
1072 	hrtimer_add_expires(timer, interval);  in hrtimer_forward()
1086 static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,  in enqueue_hrtimer()  argument
1089 	debug_activate(timer, mode);  in enqueue_hrtimer()
1095 	WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED);  in enqueue_hrtimer()
1097 	return timerqueue_add(&base->active, &timer->node);  in enqueue_hrtimer()
1110 static void __remove_hrtimer(struct hrtimer *timer,  in __remove_hrtimer()  argument
1115 	u8 state = timer->state;  in __remove_hrtimer()
1118 	WRITE_ONCE(timer->state, newstate);  in __remove_hrtimer()
1122 	if (!timerqueue_del(&base->active, &timer->node))  in __remove_hrtimer()
1133 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1141 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,  in remove_hrtimer()  argument
1144 	u8 state = timer->state;  in remove_hrtimer()
1157 		debug_deactivate(timer);  in remove_hrtimer()
1171 		__remove_hrtimer(timer, base, state, reprogram);  in remove_hrtimer()
1177 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,  in hrtimer_update_lowres()  argument
1186 	timer->is_rel = mode & HRTIMER_MODE_REL;  in hrtimer_update_lowres()
1187 	if (timer->is_rel)  in hrtimer_update_lowres()
1218 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,  in __hrtimer_start_range_ns()  argument
1235 	force_local &= base->cpu_base->next_timer == timer;  in __hrtimer_start_range_ns()
1254 	remove_hrtimer(timer, base, true, force_local);  in __hrtimer_start_range_ns()
1259 	tim = hrtimer_update_lowres(timer, tim, mode);  in __hrtimer_start_range_ns()
1261 	hrtimer_set_expires_range_ns(timer, tim, delta_ns);  in __hrtimer_start_range_ns()
1265 		new_base = switch_hrtimer_base(timer, base,  in __hrtimer_start_range_ns()
1271 	first = enqueue_hrtimer(timer, new_base, mode);  in __hrtimer_start_range_ns()
1312 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,  in hrtimer_start_range_ns()  argument
1324 		WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);  in hrtimer_start_range_ns()
1326 		WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);  in hrtimer_start_range_ns()
1328 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
1330 	if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))  in hrtimer_start_range_ns()
1331 		hrtimer_reprogram(timer, true);  in hrtimer_start_range_ns()
1333 	unlock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
1348 int hrtimer_try_to_cancel(struct hrtimer *timer)  in hrtimer_try_to_cancel()  argument
1360 	if (!hrtimer_active(timer))  in hrtimer_try_to_cancel()
1363 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1365 	if (!hrtimer_callback_running(timer))  in hrtimer_try_to_cancel()
1366 		ret = remove_hrtimer(timer, base, false, false);  in hrtimer_try_to_cancel()
1368 	unlock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1439 void hrtimer_cancel_wait_running(const struct hrtimer *timer)  in hrtimer_cancel_wait_running()  argument
1442 	struct hrtimer_clock_base *base = READ_ONCE(timer->base);  in hrtimer_cancel_wait_running()
1448 	if (!timer->is_soft || is_migration_base(base)) {  in hrtimer_cancel_wait_running()
1484 int hrtimer_cancel(struct hrtimer *timer)  in hrtimer_cancel()  argument
1489 		ret = hrtimer_try_to_cancel(timer);  in hrtimer_cancel()
1492 			hrtimer_cancel_wait_running(timer);  in hrtimer_cancel()
1503 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)  in __hrtimer_get_remaining()  argument
1508 	lock_hrtimer_base(timer, &flags);  in __hrtimer_get_remaining()
1510 		rem = hrtimer_expires_remaining_adjusted(timer);  in __hrtimer_get_remaining()
1512 		rem = hrtimer_expires_remaining(timer);  in __hrtimer_get_remaining()
1513 	unlock_hrtimer_base(timer, &flags);  in __hrtimer_get_remaining()
1609 ktime_t hrtimer_cb_get_time(const struct hrtimer *timer)  in hrtimer_cb_get_time()  argument
1611 	return __hrtimer_cb_get_time(timer->base->clockid);  in hrtimer_cb_get_time()
1615 static void __hrtimer_setup(struct hrtimer *timer,  in __hrtimer_setup()  argument
1632 	memset(timer, 0, sizeof(struct hrtimer));  in __hrtimer_setup()
1646 	timer->is_soft = softtimer;  in __hrtimer_setup()
1647 	timer->is_hard = !!(mode & HRTIMER_MODE_HARD);  in __hrtimer_setup()
1648 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_setup()
1649 	timerqueue_init(&timer->node);  in __hrtimer_setup()
1652 		ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout;  in __hrtimer_setup()
1654 		ACCESS_PRIVATE(timer, function) = function;  in __hrtimer_setup()
1670 void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),  in hrtimer_setup()  argument
1673 	debug_setup(timer, clock_id, mode);  in hrtimer_setup()
1674 	__hrtimer_setup(timer, function, clock_id, mode);  in hrtimer_setup()
1688 void hrtimer_setup_on_stack(struct hrtimer *timer,  in hrtimer_setup_on_stack()  argument
1692 	debug_setup_on_stack(timer, clock_id, mode);  in hrtimer_setup_on_stack()
1693 	__hrtimer_setup(timer, function, clock_id, mode);  in hrtimer_setup_on_stack()
1704 bool hrtimer_active(const struct hrtimer *timer)  in hrtimer_active()  argument
1710 		base = READ_ONCE(timer->base);  in hrtimer_active()
1713 		if (timer->state != HRTIMER_STATE_INACTIVE ||  in hrtimer_active()
1714 		    base->running == timer)  in hrtimer_active()
1718 		 base != READ_ONCE(timer->base));  in hrtimer_active()
1744 			  struct hrtimer *timer, ktime_t *now,  in __run_hrtimer()  argument
1753 	debug_hrtimer_deactivate(timer);  in __run_hrtimer()
1754 	base->running = timer;  in __run_hrtimer()
1765 	__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);  in __run_hrtimer()
1766 	fn = ACCESS_PRIVATE(timer, function);  in __run_hrtimer()
1774 		timer->is_rel = false;  in __run_hrtimer()
1782 	trace_hrtimer_expire_entry(timer, now);  in __run_hrtimer()
1783 	expires_in_hardirq = lockdep_hrtimer_enter(timer);  in __run_hrtimer()
1785 	restart = fn(timer);  in __run_hrtimer()
1788 	trace_hrtimer_expire_exit(timer);  in __run_hrtimer()
1801 	    !(timer->state & HRTIMER_STATE_ENQUEUED))  in __run_hrtimer()
1802 		enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);  in __run_hrtimer()
1813 	WARN_ON_ONCE(base->running != timer);  in __run_hrtimer()
1830 			struct hrtimer *timer;  in __hrtimer_run_queues()  local
1832 			timer = container_of(node, struct hrtimer, node);  in __hrtimer_run_queues()
1846 			if (basenow < hrtimer_get_softexpires(timer))  in __hrtimer_run_queues()
1849 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
2013 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)  in hrtimer_wakeup()  argument
2016 		container_of(timer, struct hrtimer_sleeper, timer);  in hrtimer_wakeup()
2044 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)  in hrtimer_sleeper_start_expires()
2047 	hrtimer_start_expires(&sl->timer, mode);  in hrtimer_sleeper_start_expires()
2078 	__hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode);  in __hrtimer_setup_sleeper()
2091 	debug_setup_on_stack(&sl->timer, clock_id, mode);  in hrtimer_setup_sleeper_on_stack()
2126 		hrtimer_cancel(&t->timer);  in do_nanosleep()
2138 		ktime_t rem = hrtimer_expires_remaining(&t->timer);  in do_nanosleep()
2156 	hrtimer_set_expires(&t.timer, restart->nanosleep.expires);  in hrtimer_nanosleep_restart()
2158 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep_restart()
2170 	hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns);  in hrtimer_nanosleep()
2182 	restart->nanosleep.clockid = t.timer.base->clockid;  in hrtimer_nanosleep()
2183 	restart->nanosleep.expires = hrtimer_get_expires(&t.timer);  in hrtimer_nanosleep()
2186 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep()
2275 	struct hrtimer *timer;  in migrate_hrtimer_list()  local
2279 		timer = container_of(node, struct hrtimer, node);  in migrate_hrtimer_list()
2280 		BUG_ON(hrtimer_callback_running(timer));  in migrate_hrtimer_list()
2281 		debug_deactivate(timer);  in migrate_hrtimer_list()
2288 		__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);  in migrate_hrtimer_list()
2289 		timer->base = new_base;  in migrate_hrtimer_list()
2298 		enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);  in migrate_hrtimer_list()