Lines Matching defs:cpu_base
61 * timer->base->cpu_base->running == timer
69 * All state transitions are protected by cpu_base->lock.
84 * cpu_base->active
157 * timer->base->cpu_base
162 .cpu_base = &migration_cpu_base,
191 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
195 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
213 * Called with cpu_base->lock of target cpu held.
239 return expires >= new_base->cpu_base->expires_next;
295 raw_spin_unlock(&base->cpu_base->lock);
296 raw_spin_lock(&new_base->cpu_base->lock);
299 raw_spin_unlock(&new_base->cpu_base->lock);
300 raw_spin_lock(&base->cpu_base->lock);
319 __acquires(&timer->base->cpu_base->lock)
323 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
532 #define for_each_active_base(base, cpu_base, active) \
535 for (base = &cpu_base->clock_base[idx]; !done; done = true)
542 * does not update cpu_base->next_timer/expires.
544 static ktime_t hrtimer_bases_next_event_without(struct hrtimer_cpu_base *cpu_base,
551 lockdep_assert_held(&cpu_base->lock);
553 for_each_active_base(base, cpu_base, active) {
587 static void hrtimer_bases_first(struct hrtimer_cpu_base *cpu_base,unsigned int active,
593 for_each_active_base(base, cpu_base, active) {
603 * Recomputes cpu_base::*next_timer and returns the earliest expires_next
604 * but does not set cpu_base::*expires_next, that is done by
606 * cpu_base::*expires_next right away, reprogramming logic would no longer
622 static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
628 lockdep_assert_held(&cpu_base->lock);
630 if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
631 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
633 hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
634 cpu_base->softirq_next_timer = next_timer;
638 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
640 hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
641 cpu_base->next_timer = next_timer;
646 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
655 if (!cpu_base->softirq_activated) {
656 soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
661 cpu_base->softirq_expires_next = soft;
664 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
666 * If a softirq timer is expiring first, update cpu_base->next_timer
670 cpu_base->next_timer = cpu_base->softirq_next_timer;
698 static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
701 cpu_base->hres_active : 0;
710 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer,
713 cpu_base->expires_next = expires_next;
732 if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
739 static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip_equal)
741 ktime_t expires_next = hrtimer_update_next_event(cpu_base);
743 if (skip_equal && expires_next == cpu_base->expires_next)
746 __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
842 * Called with interrupts disabled and base->cpu_base.lock held
846 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
868 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
887 if (base->cpu_base != cpu_base)
890 if (expires >= cpu_base->expires_next)
894 if (cpu_base->deferred_rearm)
897 cpu_base->next_timer = timer;
899 __hrtimer_reprogram(cpu_base, timer, expires);
902 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
918 seq = cpu_base->clock_was_set_seq;
919 hrtimer_update_base(cpu_base);
925 if (seq == cpu_base->clock_was_set_seq)
929 if (cpu_base->deferred_rearm) {
930 cpu_base->deferred_needs_update = true;
937 * @cpu_base. If so, the IPI must be invoked because per CPU clock
940 active &= cpu_base->active_bases;
942 for_each_active_base(base, cpu_base, active) {
947 if (expires < cpu_base->expires_next)
953 if (cpu_base->softirq_activated)
955 if (expires < cpu_base->softirq_expires_next)
992 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
994 guard(raw_spinlock_irqsave)(&cpu_base->lock);
995 if (update_needs_ipi(cpu_base, bases))
1037 __releases(&timer->base->cpu_base->lock)
1039 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
1107 lockdep_assert_held(&base->cpu_base->lock);
1110 WARN_ON_ONCE(!base->cpu_base->online);
1112 base->cpu_base->active_bases |= 1 << base->index;
1142 struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1145 lockdep_assert_held(&cpu_base->lock);
1156 cpu_base->active_bases &= ~(1 << base->index);
1165 * If reprogram is false don't update cpu_base->next_timer and do not
1173 if (!reprogram || timer != cpu_base->next_timer || timer->is_lazy)
1176 if (cpu_base->deferred_rearm)
1177 cpu_base->deferred_needs_update = true;
1179 hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
1185 lockdep_assert_held(&base->cpu_base->lock);
1200 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1260 base->cpu_base->active_bases |= 1 << base->index;
1293 static void hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
1295 ktime_t expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
1300 * hrtimer. cpu_base->softirq_expires_next needs to be updated!
1306 * cpu_base->next_timer is recomputed by __hrtimer_get_next_event()
1307 * cpu_base->expires_next is only set by hrtimer_reprogram()
1309 hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
1360 struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1362 was_first = cpu_base->next_timer == timer;
1370 bool is_local = cpu_base == this_cpu_base;
1405 cpu_base = base->cpu_base;
1411 if (cpu_base->deferred_rearm) {
1412 cpu_base->deferred_needs_update = true;
1416 if (!was_first || cpu_base != this_cpu_base) {
1434 smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd);
1448 if (cpu_base->expires_next <= hrtimer_get_expires(timer))
1458 hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
1560 * If there is a waiter for cpu_base->expiry_lock, then it was waiting for
1564 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, unsigned long flags)
1566 if (atomic_read(&cpu_base->timer_waiters)) {
1567 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1568 spin_unlock(&cpu_base->softirq_expiry_lock);
1569 spin_lock(&cpu_base->softirq_expiry_lock);
1570 raw_spin_lock_irq(&cpu_base->lock);
1623 atomic_inc(&base->cpu_base->timer_waiters);
1624 spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
1625 atomic_dec(&base->cpu_base->timer_waiters);
1626 spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
1686 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1689 guard(raw_spinlock_irqsave)(&cpu_base->lock);
1690 if (!hrtimer_hres_active(cpu_base))
1691 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1705 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1709 guard(raw_spinlock_irqsave)(&cpu_base->lock);
1710 if (!hrtimer_hres_active(cpu_base))
1713 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1714 if (active && !cpu_base->softirq_activated)
1715 expires = hrtimer_bases_next_event_without(cpu_base, exclude, active, KTIME_MAX);
1717 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1720 return hrtimer_bases_next_event_without(cpu_base, exclude, active, expires);
1768 struct hrtimer_cpu_base *cpu_base;
1782 cpu_base = raw_cpu_ptr(&hrtimer_bases);
1797 timer->base = &cpu_base->clock_base[base];
1879 * On the read side we ensure we observe timer->is_queued and cpu_base->running
1888 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_clock_base *base,
1890 __must_hold(&cpu_base->lock)
1896 lockdep_assert_held(&cpu_base->lock);
1926 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1934 raw_spin_lock_irq(&cpu_base->lock);
1941 * Note: Because we dropped the cpu_base->lock above,
1968 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
1971 unsigned int active = cpu_base->active_bases & active_mask;
1974 for_each_active_base(base, cpu_base, active) {
1994 __run_hrtimer(cpu_base, base, timer, basenow, flags);
1996 hrtimer_sync_wait_running(cpu_base, flags);
2003 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2007 hrtimer_cpu_base_lock_expiry(cpu_base);
2008 raw_spin_lock_irqsave(&cpu_base->lock, flags);
2010 now = hrtimer_update_base(cpu_base);
2011 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
2013 cpu_base->softirq_activated = false;
2014 hrtimer_update_softirq_timer(cpu_base, true);
2016 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2017 hrtimer_cpu_base_unlock_expiry(cpu_base);
2026 static void hrtimer_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next, bool deferred)
2028 cpu_base->expires_next = expires_next;
2029 cpu_base->deferred_rearm = false;
2031 if (unlikely(cpu_base->hang_detected)) {
2037 min(100 * NSEC_PER_MSEC, cpu_base->max_hang_time));
2045 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2048 if (!cpu_base->deferred_rearm)
2051 guard(raw_spinlock)(&cpu_base->lock);
2052 if (cpu_base->deferred_needs_update) {
2053 hrtimer_update_base(cpu_base);
2054 expires_next = hrtimer_update_next_event(cpu_base);
2057 expires_next = cpu_base->deferred_expires_next;
2059 hrtimer_rearm(cpu_base, expires_next, true);
2063 hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
2066 cpu_base->deferred_needs_update = false;
2068 cpu_base->deferred_expires_next = expires_next;
2073 hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
2075 hrtimer_rearm(cpu_base, expires_next, false);
2085 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2090 BUG_ON(!cpu_base->hres_active);
2091 cpu_base->nr_events++;
2095 raw_spin_lock_irqsave(&cpu_base->lock, flags);
2096 entry_time = now = hrtimer_update_base(cpu_base);
2098 cpu_base->deferred_rearm = true;
2105 cpu_base->expires_next = KTIME_MAX;
2107 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
2108 cpu_base->softirq_expires_next = KTIME_MAX;
2109 cpu_base->softirq_activated = true;
2113 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
2125 now = hrtimer_update_base(cpu_base);
2126 expires_next = hrtimer_update_next_event(cpu_base);
2127 cpu_base->hang_detected = false;
2133 cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta);
2134 cpu_base->nr_hangs++;
2135 cpu_base->hang_detected = true;
2138 hrtimer_interrupt_rearm(cpu_base, expires_next);
2139 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2149 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2153 if (hrtimer_hres_active(cpu_base))
2168 raw_spin_lock_irqsave(&cpu_base->lock, flags);
2169 now = hrtimer_update_base(cpu_base);
2171 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
2172 cpu_base->softirq_expires_next = KTIME_MAX;
2173 cpu_base->softirq_activated = true;
2177 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
2178 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2404 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
2407 struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
2409 clock_b->cpu_base = cpu_base;
2410 seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
2414 cpu_base->cpu = cpu;
2415 hrtimer_cpu_base_init_expiry_lock(cpu_base);
2421 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2424 cpu_base->active_bases = 0;
2425 cpu_base->hres_active = false;
2426 cpu_base->hang_detected = false;
2427 cpu_base->next_timer = NULL;
2428 cpu_base->softirq_next_timer = NULL;
2429 cpu_base->expires_next = KTIME_MAX;
2430 cpu_base->softirq_expires_next = KTIME_MAX;
2431 cpu_base->softirq_activated = false;
2432 cpu_base->online = true;