142 		.cpu_base = &migration_cpu_base,
172 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
176 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
220 	return expires >= new_base->cpu_base->expires_next;  in hrtimer_suitable_target()
278 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
279 		raw_spin_lock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
283 			raw_spin_unlock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
284 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
303 	__acquires(&timer->base->cpu_base->lock)  in lock_hrtimer_base()
307 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
496 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)  in __next_base()  argument
506 	return &cpu_base->clock_base[idx];  in __next_base()
509 #define for_each_active_base(base, cpu_base, active)	\  argument
510 	while ((base = __next_base((cpu_base), &(active))))
512 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_next_event_base()  argument
520 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_next_event_base()
543 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
545 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
579 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)  in __hrtimer_get_next_event()  argument
585 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {  in __hrtimer_get_next_event()
586 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in __hrtimer_get_next_event()
587 		cpu_base->softirq_next_timer = NULL;  in __hrtimer_get_next_event()
588 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,  in __hrtimer_get_next_event()
591 		next_timer = cpu_base->softirq_next_timer;  in __hrtimer_get_next_event()
595 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in __hrtimer_get_next_event()
596 		cpu_base->next_timer = next_timer;  in __hrtimer_get_next_event()
597 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,  in __hrtimer_get_next_event()
604 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)  in hrtimer_update_next_event()  argument
613 	if (!cpu_base->softirq_activated) {  in hrtimer_update_next_event()
614 		soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_next_event()
619 		cpu_base->softirq_expires_next = soft;  in hrtimer_update_next_event()
622 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);  in hrtimer_update_next_event()
628 		cpu_base->next_timer = cpu_base->softirq_next_timer;  in hrtimer_update_next_event()
654 static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)  in hrtimer_hres_active()  argument
657 		cpu_base->hres_active : 0;  in hrtimer_hres_active()
660 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_reprogram()  argument
664 	cpu_base->expires_next = expires_next;  in __hrtimer_reprogram()
683 	if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in __hrtimer_reprogram()
695 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)  in hrtimer_force_reprogram()  argument
699 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_force_reprogram()
701 	if (skip_equal && expires_next == cpu_base->expires_next)  in hrtimer_force_reprogram()
704 	__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);  in hrtimer_force_reprogram()
813 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_reprogram()  local
834 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;  in hrtimer_reprogram()
854 	if (base->cpu_base != cpu_base)  in hrtimer_reprogram()
857 	if (expires >= cpu_base->expires_next)  in hrtimer_reprogram()
864 	if (cpu_base->in_hrtirq)  in hrtimer_reprogram()
867 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
869 	__hrtimer_reprogram(cpu_base, timer, expires);  in hrtimer_reprogram()
872 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,  in update_needs_ipi()  argument
889 	seq = cpu_base->clock_was_set_seq;  in update_needs_ipi()
890 	hrtimer_update_base(cpu_base);  in update_needs_ipi()
896 	if (seq == cpu_base->clock_was_set_seq)  in update_needs_ipi()
904 	if (cpu_base->in_hrtirq)  in update_needs_ipi()
913 	active &= cpu_base->active_bases;  in update_needs_ipi()
915 	for_each_active_base(base, cpu_base, active) {  in update_needs_ipi()
920 		if (expires < cpu_base->expires_next)  in update_needs_ipi()
926 		if (cpu_base->softirq_activated)  in update_needs_ipi()
928 		if (expires < cpu_base->softirq_expires_next)  in update_needs_ipi()
950 	struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);  in clock_was_set()  local
954 	if (!hrtimer_hres_active(cpu_base) && !tick_nohz_is_active())  in clock_was_set()
967 		cpu_base = &per_cpu(hrtimer_bases, cpu);  in clock_was_set()
968 		raw_spin_lock_irqsave(&cpu_base->lock, flags);  in clock_was_set()
970 		if (update_needs_ipi(cpu_base, bases))  in clock_was_set()
973 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in clock_was_set()
1019 	__releases(&timer->base->cpu_base->lock)  in unlock_hrtimer_base()
1021 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
1090 	WARN_ON_ONCE(!base->cpu_base->online);  in enqueue_hrtimer()
1092 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
1114 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __remove_hrtimer()  local
1123 		cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
1133 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1134 		hrtimer_force_reprogram(cpu_base, 1);  in __remove_hrtimer()
1158 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
1194 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)  in hrtimer_update_softirq_timer()  argument
1201 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_softirq_timer()
1215 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);  in hrtimer_update_softirq_timer()
1234 	force_local = base->cpu_base == this_cpu_base;  in __hrtimer_start_range_ns()
1235 	force_local &= base->cpu_base->next_timer == timer;  in __hrtimer_start_range_ns()
1287 			struct hrtimer_cpu_base *new_cpu_base = new_base->cpu_base;  in __hrtimer_start_range_ns()
1299 	hrtimer_force_reprogram(new_base->cpu_base, 1);  in __hrtimer_start_range_ns()
1400 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,  in hrtimer_sync_wait_running()  argument
1403 	if (atomic_read(&cpu_base->timer_waiters)) {  in hrtimer_sync_wait_running()
1404 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_sync_wait_running()
1405 		spin_unlock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1406 		spin_lock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1407 		raw_spin_lock_irq(&cpu_base->lock);  in hrtimer_sync_wait_running()
1460 	atomic_inc(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1461 	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1462 	atomic_dec(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1463 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1527 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_get_next_event()  local
1531 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1533 	if (!hrtimer_hres_active(cpu_base))  in hrtimer_get_next_event()
1534 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_get_next_event()
1536 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1550 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_next_event_without()  local
1554 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1556 	if (hrtimer_hres_active(cpu_base)) {  in hrtimer_next_event_without()
1559 		if (!cpu_base->softirq_activated) {  in hrtimer_next_event_without()
1560 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in hrtimer_next_event_without()
1561 			expires = __hrtimer_next_event_base(cpu_base, exclude,  in hrtimer_next_event_without()
1564 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in hrtimer_next_event_without()
1565 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,  in hrtimer_next_event_without()
1569 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1620 	struct hrtimer_cpu_base *cpu_base;  in __hrtimer_setup()  local
1634 	cpu_base = raw_cpu_ptr(&hrtimer_bases);  in __hrtimer_setup()
1648 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_setup()
1742 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,  in __run_hrtimer()  argument
1745 			  unsigned long flags) __must_hold(&cpu_base->lock)  in __run_hrtimer()
1751 	lockdep_assert_held(&cpu_base->lock);  in __run_hrtimer()
1781 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in __run_hrtimer()
1789 	raw_spin_lock_irq(&cpu_base->lock);  in __run_hrtimer()
1817 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,  in __hrtimer_run_queues()  argument
1821 	unsigned int active = cpu_base->active_bases & active_mask;  in __hrtimer_run_queues()
1823 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_run_queues()
1849 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1851 				hrtimer_sync_wait_running(cpu_base, flags);  in __hrtimer_run_queues()
1858 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_softirq()  local
1862 	hrtimer_cpu_base_lock_expiry(cpu_base);  in hrtimer_run_softirq()
1863 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1865 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_softirq()
1866 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);  in hrtimer_run_softirq()
1868 	cpu_base->softirq_activated = 0;  in hrtimer_run_softirq()
1869 	hrtimer_update_softirq_timer(cpu_base, true);  in hrtimer_run_softirq()
1871 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1872 	hrtimer_cpu_base_unlock_expiry(cpu_base);  in hrtimer_run_softirq()
1883 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_interrupt()  local
1888 	BUG_ON(!cpu_base->hres_active);  in hrtimer_interrupt()
1889 	cpu_base->nr_events++;  in hrtimer_interrupt()
1892 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1893 	entry_time = now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1895 	cpu_base->in_hrtirq = 1;  in hrtimer_interrupt()
1903 	cpu_base->expires_next = KTIME_MAX;  in hrtimer_interrupt()
1905 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_interrupt()
1906 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_interrupt()
1907 		cpu_base->softirq_activated = 1;  in hrtimer_interrupt()
1911 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_interrupt()
1914 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_interrupt()
1919 	cpu_base->expires_next = expires_next;  in hrtimer_interrupt()
1920 	cpu_base->in_hrtirq = 0;  in hrtimer_interrupt()
1921 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1925 		cpu_base->hang_detected = 0;  in hrtimer_interrupt()
1942 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1943 	now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1944 	cpu_base->nr_retries++;  in hrtimer_interrupt()
1953 	cpu_base->nr_hangs++;  in hrtimer_interrupt()
1954 	cpu_base->hang_detected = 1;  in hrtimer_interrupt()
1955 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1958 	if ((unsigned int)delta > cpu_base->max_hang_time)  in hrtimer_interrupt()
1959 		cpu_base->max_hang_time = (unsigned int) delta;  in hrtimer_interrupt()
1978 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_queues()  local
1982 	if (hrtimer_hres_active(cpu_base))  in hrtimer_run_queues()
1997 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_queues()
1998 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_queues()
2000 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_run_queues()
2001 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_run_queues()
2002 		cpu_base->softirq_activated = 1;  in hrtimer_run_queues()
2006 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_run_queues()
2007 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_queues()
2238 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);  in hrtimers_prepare_cpu()  local
2242 		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];  in hrtimers_prepare_cpu()
2244 		clock_b->cpu_base = cpu_base;  in hrtimers_prepare_cpu()
2245 		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);  in hrtimers_prepare_cpu()
2249 	cpu_base->cpu = cpu;  in hrtimers_prepare_cpu()
2250 	hrtimer_cpu_base_init_expiry_lock(cpu_base);  in hrtimers_prepare_cpu()
2256 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimers_cpu_starting()  local
2259 	cpu_base->active_bases = 0;  in hrtimers_cpu_starting()
2260 	cpu_base->hres_active = 0;  in hrtimers_cpu_starting()
2261 	cpu_base->hang_detected = 0;  in hrtimers_cpu_starting()
2262 	cpu_base->next_timer = NULL;  in hrtimers_cpu_starting()
2263 	cpu_base->softirq_next_timer = NULL;  in hrtimers_cpu_starting()
2264 	cpu_base->expires_next = KTIME_MAX;  in hrtimers_cpu_starting()
2265 	cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimers_cpu_starting()
2266 	cpu_base->online = 1;  in hrtimers_cpu_starting()