Lines Matching refs:timer
16 * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
58 #include <trace/events/timer.h>
65 * The timer wheel has LVL_DEPTH array levels. Each level provides an array of
72 * The array level of a newly armed timer depends on the relative expiry
76 * Contrary to the original timer wheel implementation, which aims for 'exact'
78 * the timers into the lower array levels. The previous 'classic' timer wheel
83 * This is an optimization of the original timer wheel implementation for the
84 * majority of the timer wheel use cases: timeouts. The vast majority of
206 * struct timer_base - Per CPU timer base (number of base depends on config)
210 * currently running timer, the pointer is set to the
211 * timer, which expires at the moment. If no timer is
214 * timer expiry callback execution and when trying to
215 * delete a running timer and it wasn't successful in
218 * caller tries to delete the running timer. It also
220 * delete a timer preempted the softirq thread which
221 * is running the timer callback function.
223 * waiting for the end of the timer callback function
225 * @clk: clock of the timer base; is updated before enqueue
226 * of a timer; during expiry, it is 1 offset ahead of
229 * @next_expiry: expiry value of the first timer; it is updated when
230 * finding the next timer and during enqueue; the
232 * @cpu: Number of CPU the timer base belongs to
234 * required. Value is set true, when a timer was
241 * @timers_pending: Is set, when a timer is pending in the base. It is only
243 * @pending_map: bitmap of the timer wheel; each bit reflects a
245 * single timer is enqueued in the related bucket.
247 * of the timer wheel. The list contains all timers
368 * due to delays of the timer irq, long irq off times etc etc) then
551 static inline unsigned int timer_get_idx(struct timer_list *timer)
553 return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;
556 static inline void timer_set_idx(struct timer_list *timer, unsigned int idx)
558 timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |
571 * The timer wheel has to guarantee that a timer does not fire
622 trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
627 * new deferrable timer is enqueued will wake up the remote CPU but
628 * nothing will be done with the deferrable timer base. Therefore skip
631 if (!is_timers_nohz_active() || timer->flags & TIMER_DEFERRABLE)
636 * timer is pinned. If it is a non pinned timer, it is only queued
637 * on the remote CPU, when timer was running during queueing. Then
643 WARN_ON_ONCE(!(timer->flags & TIMER_PINNED ||
650 * Enqueue the timer into the hash bucket, mark it pending in
651 * the bitmap, store the index in the timer flags then wake up
654 static void enqueue_timer(struct timer_base *base, struct timer_list *timer,
658 hlist_add_head(&timer->entry, base->vectors + idx);
660 timer_set_idx(timer, idx);
662 trace_timer_start(timer, bucket_expiry);
665 * Check whether this is the new first expiring timer. The
666 * effective expiry time of the timer is required here
667 * (bucket_expiry) instead of timer->expires.
677 trigger_dyntick_cpu(base, timer);
681 static void internal_add_timer(struct timer_base *base, struct timer_list *timer)
686 idx = calc_wheel_index(timer->expires, base->clk, &bucket_expiry);
687 enqueue_timer(base, timer, idx, bucket_expiry);
708 struct delayed_work, timer, work.func),
710 struct kthread_delayed_work, timer, work.func),
715 struct timer_list *timer = addr;
719 if (timer_hints[i].function == timer->function) {
726 return timer->function;
731 struct timer_list *timer = addr;
733 return (timer->entry.pprev == NULL &&
734 timer->entry.next == TIMER_ENTRY_STATIC);
743 struct timer_list *timer = addr;
747 del_timer_sync(timer);
748 debug_object_init(timer, &timer_debug_descr);
755 /* Stub timer callback for improperly used timers. */
768 struct timer_list *timer = addr;
772 timer_setup(timer, stub_timer, 0);
789 struct timer_list *timer = addr;
793 del_timer_sync(timer);
794 debug_object_free(timer, &timer_debug_descr);
807 struct timer_list *timer = addr;
811 timer_setup(timer, stub_timer, 0);
828 static inline void debug_timer_init(struct timer_list *timer)
830 debug_object_init(timer, &timer_debug_descr);
833 static inline void debug_timer_activate(struct timer_list *timer)
835 debug_object_activate(timer, &timer_debug_descr);
838 static inline void debug_timer_deactivate(struct timer_list *timer)
840 debug_object_deactivate(timer, &timer_debug_descr);
843 static inline void debug_timer_assert_init(struct timer_list *timer)
845 debug_object_assert_init(timer, &timer_debug_descr);
848 static void do_init_timer(struct timer_list *timer,
853 void init_timer_on_stack_key(struct timer_list *timer,
858 debug_object_init_on_stack(timer, &timer_debug_descr);
859 do_init_timer(timer, func, flags, name, key);
863 void destroy_timer_on_stack(struct timer_list *timer)
865 debug_object_free(timer, &timer_debug_descr);
870 static inline void debug_timer_init(struct timer_list *timer) { }
871 static inline void debug_timer_activate(struct timer_list *timer) { }
872 static inline void debug_timer_deactivate(struct timer_list *timer) { }
873 static inline void debug_timer_assert_init(struct timer_list *timer) { }
876 static inline void debug_init(struct timer_list *timer)
878 debug_timer_init(timer);
879 trace_timer_init(timer);
882 static inline void debug_deactivate(struct timer_list *timer)
884 debug_timer_deactivate(timer);
885 trace_timer_cancel(timer);
888 static inline void debug_assert_init(struct timer_list *timer)
890 debug_timer_assert_init(timer);
893 static void do_init_timer(struct timer_list *timer,
898 timer->entry.pprev = NULL;
899 timer->function = func;
902 timer->flags = flags | raw_smp_processor_id();
903 lockdep_init_map(&timer->lockdep_map, name, key, 0);
907 * init_timer_key - initialize a timer
908 * @timer: the timer to be initialized
909 * @func: timer callback function
910 * @flags: timer flags
911 * @name: name of the timer
912 * @key: lockdep class key of the fake lock used for tracking timer
915 * init_timer_key() must be done to a timer prior to calling *any* of the
916 * other timer functions.
918 void init_timer_key(struct timer_list *timer,
922 debug_init(timer);
923 do_init_timer(timer, func, flags, name, key);
927 static inline void detach_timer(struct timer_list *timer, bool clear_pending)
929 struct hlist_node *entry = &timer->entry;
931 debug_deactivate(timer);
939 static int detach_if_pending(struct timer_list *timer, struct timer_base *base,
942 unsigned idx = timer_get_idx(timer);
944 if (!timer_pending(timer))
947 if (hlist_is_singular_node(&timer->entry, base->vectors + idx)) {
952 detach_timer(timer, clear_pending);
961 * If the timer is deferrable and NO_HZ_COMMON is set then we need
975 * If the timer is deferrable and NO_HZ_COMMON is set then we need
1026 * When a timer is migrating then the TIMER_MIGRATING flag is set and we need
1029 static struct timer_base *lock_timer_base(struct timer_list *timer,
1031 __acquires(timer->base->lock)
1042 tf = READ_ONCE(timer->flags);
1047 if (timer->flags == tf)
1060 __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)
1067 debug_assert_init(timer);
1071 * the timer is re-modified to have the same timeout or ends up in the
1074 if (!(options & MOD_TIMER_NOTPENDING) && timer_pending(timer)) {
1078 * timer with this expiry.
1080 long diff = timer->expires - expires;
1088 * We lock timer base and calculate the bucket index right
1089 * here. If the timer ends up in the same bucket, then we
1093 base = lock_timer_base(timer, &flags);
1095 * Has @timer been shutdown? This needs to be evaluated
1099 if (!timer->function)
1104 if (timer_pending(timer) && (options & MOD_TIMER_REDUCE) &&
1105 time_before_eq(timer->expires, expires)) {
1115 * timer. If it matches set the expiry to the new value so a
1118 if (idx == timer_get_idx(timer)) {
1120 timer->expires = expires;
1121 else if (time_after(timer->expires, expires))
1122 timer->expires = expires;
1127 base = lock_timer_base(timer, &flags);
1129 * Has @timer been shutdown? This needs to be evaluated
1133 if (!timer->function)
1139 ret = detach_if_pending(timer, base, false);
1143 new_base = get_timer_this_cpu_base(timer->flags);
1147 * We are trying to schedule the timer on the new base.
1148 * However we can't change timer's base while it is running,
1149 * otherwise timer_delete_sync() can't detect that the timer's
1151 * timer is serialized wrt itself.
1153 if (likely(base->running_timer != timer)) {
1155 timer->flags |= TIMER_MIGRATING;
1160 WRITE_ONCE(timer->flags,
1161 (timer->flags & ~TIMER_BASEMASK) | base->cpu);
1166 debug_timer_activate(timer);
1168 timer->expires = expires;
1176 enqueue_timer(base, timer, idx, bucket_expiry);
1178 internal_add_timer(base, timer);
1187 * mod_timer_pending - Modify a pending timer's timeout
1188 * @timer: The pending timer to be modified
1194 * If @timer->function == NULL then the start operation is silently
1198 * * %0 - The timer was inactive and not modified or was in
1200 * * %1 - The timer was active and requeued to expire at @expires
1202 int mod_timer_pending(struct timer_list *timer, unsigned long expires)
1204 return __mod_timer(timer, expires, MOD_TIMER_PENDING_ONLY);
1209 * mod_timer - Modify a timer's timeout
1210 * @timer: The timer to be modified
1213 * mod_timer(timer, expires) is equivalent to:
1215 * del_timer(timer); timer->expires = expires; add_timer(timer);
1218 * case that the timer is inactive, the del_timer() part is a NOP. The
1219 * timer is in any case activated with the new expiry time @expires.
1222 * same timer, then mod_timer() is the only safe way to modify the timeout,
1223 * since add_timer() cannot modify an already running timer.
1225 * If @timer->function == NULL then the start operation is silently
1229 * * %0 - The timer was inactive and started or was in shutdown
1231 * * %1 - The timer was active and requeued to expire at @expires or
1232 * the timer was active and not modified because @expires did
1235 int mod_timer(struct timer_list *timer, unsigned long expires)
1237 return __mod_timer(timer, expires, 0);
1242 * timer_reduce - Modify a timer's timeout if it would reduce the timeout
1243 * @timer: The timer to be modified
1247 * modify an enqueued timer if that would reduce the expiration time. If
1248 * @timer is not enqueued it starts the timer.
1250 * If @timer->function == NULL then the start operation is silently
1254 * * %0 - The timer was inactive and started or was in shutdown
1256 * * %1 - The timer was active and requeued to expire at @expires or
1257 * the timer was active and not modified because @expires
1259 * timer would expire earlier than already scheduled
1261 int timer_reduce(struct timer_list *timer, unsigned long expires)
1263 return __mod_timer(timer, expires, MOD_TIMER_REDUCE);
1268 * add_timer - Start a timer
1269 * @timer: The timer to be started
1271 * Start @timer to expire at @timer->expires in the future. @timer->expires
1272 * is the absolute expiry time measured in 'jiffies'. When the timer expires
1273 * timer->function(timer) will be invoked from soft interrupt context.
1275 * The @timer->expires and @timer->function fields must be set prior
1278 * If @timer->function == NULL then the start operation is silently
1281 * If @timer->expires is already in the past @timer will be queued to
1282 * expire at the next timer tick.
1284 * This can only operate on an inactive timer. Attempts to invoke this on
1285 * an active timer are rejected with a warning.
1287 void add_timer(struct timer_list *timer)
1289 if (WARN_ON_ONCE(timer_pending(timer)))
1291 __mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
1296 * add_timer_local() - Start a timer on the local CPU
1297 * @timer: The timer to be started
1299 * Same as add_timer() except that the timer flag TIMER_PINNED is set.
1303 void add_timer_local(struct timer_list *timer)
1305 if (WARN_ON_ONCE(timer_pending(timer)))
1307 timer->flags |= TIMER_PINNED;
1308 __mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
1313 * add_timer_global() - Start a timer without TIMER_PINNED flag set
1314 * @timer: The timer to be started
1316 * Same as add_timer() except that the timer flag TIMER_PINNED is unset.
1320 void add_timer_global(struct timer_list *timer)
1322 if (WARN_ON_ONCE(timer_pending(timer)))
1324 timer->flags &= ~TIMER_PINNED;
1325 __mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
1330 * add_timer_on - Start a timer on a particular CPU
1331 * @timer: The timer to be started
1334 * Same as add_timer() except that it starts the timer on the given CPU and
1335 * the TIMER_PINNED flag is set. When timer shouldn't be a pinned timer in
1341 void add_timer_on(struct timer_list *timer, int cpu)
1346 debug_assert_init(timer);
1348 if (WARN_ON_ONCE(timer_pending(timer)))
1351 /* Make sure timer flags have TIMER_PINNED flag set */
1352 timer->flags |= TIMER_PINNED;
1354 new_base = get_timer_cpu_base(timer->flags, cpu);
1357 * If @timer was on a different CPU, it should be migrated with the
1361 base = lock_timer_base(timer, &flags);
1363 * Has @timer been shutdown? This needs to be evaluated while
1366 if (!timer->function)
1370 timer->flags |= TIMER_MIGRATING;
1375 WRITE_ONCE(timer->flags,
1376 (timer->flags & ~TIMER_BASEMASK) | cpu);
1380 debug_timer_activate(timer);
1381 internal_add_timer(base, timer);
1388 * __timer_delete - Internal function: Deactivate a timer
1389 * @timer: The timer to be deactivated
1390 * @shutdown: If true, this indicates that the timer is about to be
1393 * If @shutdown is true then @timer->function is set to NULL under the
1394 * timer base lock which prevents further rearming of the time. In that
1395 * case any attempt to rearm @timer after this function returns will be
1399 * * %0 - The timer was not pending
1400 * * %1 - The timer was pending and deactivated
1402 static int __timer_delete(struct timer_list *timer, bool shutdown)
1408 debug_assert_init(timer);
1412 * timer is pending or not to protect against a concurrent rearm
1415 * enqueued timer is dequeued and cannot end up with
1416 * timer->function == NULL in the expiry code.
1418 * If timer->function is currently executed, then this makes sure
1419 * that the callback cannot requeue the timer.
1421 if (timer_pending(timer) || shutdown) {
1422 base = lock_timer_base(timer, &flags);
1423 ret = detach_if_pending(timer, base, true);
1425 timer->function = NULL;
1433 * timer_delete - Deactivate a timer
1434 * @timer: The timer to be deactivated
1436 * The function only deactivates a pending timer, but contrary to
1437 * timer_delete_sync() it does not take into account whether the timer's
1439 * It neither prevents rearming of the timer. If @timer can be rearmed
1443 * * %0 - The timer was not pending
1444 * * %1 - The timer was pending and deactivated
1446 int timer_delete(struct timer_list *timer)
1448 return __timer_delete(timer, false);
1453 * timer_shutdown - Deactivate a timer and prevent rearming
1454 * @timer: The timer to be deactivated
1456 * The function does not wait for an eventually running timer callback on a
1457 * different CPU but it prevents rearming of the timer. Any attempt to arm
1458 * @timer after this function returns will be silently ignored.
1464 * * %0 - The timer was not pending
1465 * * %1 - The timer was pending
1467 int timer_shutdown(struct timer_list *timer)
1469 return __timer_delete(timer, true);
1474 * __try_to_del_timer_sync - Internal function: Try to deactivate a timer
1475 * @timer: Timer to deactivate
1476 * @shutdown: If true, this indicates that the timer is about to be
1479 * If @shutdown is true then @timer->function is set to NULL under the
1480 * timer base lock which prevents further rearming of the timer. Any
1481 * attempt to rearm @timer after this function returns will be silently
1484 * This function cannot guarantee that the timer cannot be rearmed
1489 * * %0 - The timer was not pending
1490 * * %1 - The timer was pending and deactivated
1491 * * %-1 - The timer callback function is running on a different CPU
1493 static int __try_to_del_timer_sync(struct timer_list *timer, bool shutdown)
1499 debug_assert_init(timer);
1501 base = lock_timer_base(timer, &flags);
1503 if (base->running_timer != timer)
1504 ret = detach_if_pending(timer, base, true);
1506 timer->function = NULL;
1514 * try_to_del_timer_sync - Try to deactivate a timer
1515 * @timer: Timer to deactivate
1517 * This function tries to deactivate a timer. On success the timer is not
1518 * queued and the timer callback function is not running on any CPU.
1520 * This function does not guarantee that the timer cannot be rearmed right
1525 * * %0 - The timer was not pending
1526 * * %1 - The timer was pending and deactivated
1527 * * %-1 - The timer callback function is running on a different CPU
1529 int try_to_del_timer_sync(struct timer_list *timer)
1531 return __try_to_del_timer_sync(timer, false);
1555 * timer callback to finish. Drop expiry_lock and reacquire it. That allows
1572 * deletion of a timer failed because the timer callback function was
1577 * delete a timer preempted the softirq thread running the timer callback
1580 static void del_timer_wait_running(struct timer_list *timer)
1584 tf = READ_ONCE(timer->flags);
1590 * which is held by the softirq across the timer
1592 * expire the next timer. In theory the timer could already
1607 static inline void del_timer_wait_running(struct timer_list *timer) { }
1611 * __timer_delete_sync - Internal function: Deactivate a timer and wait
1613 * @timer: The timer to be deactivated
1614 * @shutdown: If true, @timer->function will be set to NULL under the
1615 * timer base lock which prevents rearming of @timer
1617 * If @shutdown is not set the timer can be rearmed later. If the timer can
1621 * If @shutdown is set then @timer->function is set to NULL under timer
1622 * base lock which prevents rearming of the timer. Any attempt to rearm
1623 * a shutdown timer is silently ignored.
1625 * If the timer should be reused after shutdown it has to be initialized
1629 * * %0 - The timer was not pending
1630 * * %1 - The timer was pending and deactivated
1632 static int __timer_delete_sync(struct timer_list *timer, bool shutdown)
1644 lock_map_acquire(&timer->lockdep_map);
1645 lock_map_release(&timer->lockdep_map);
1652 WARN_ON(in_hardirq() && !(timer->flags & TIMER_IRQSAFE));
1658 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(timer->flags & TIMER_IRQSAFE))
1662 ret = __try_to_del_timer_sync(timer, shutdown);
1665 del_timer_wait_running(timer);
1674 * timer_delete_sync - Deactivate a timer and wait for the handler to finish.
1675 * @timer: The timer to be deactivated
1677 * Synchronization rules: Callers must prevent restarting of the timer,
1679 * interrupt contexts unless the timer is an irqsafe one. The caller must
1680 * not hold locks which would prevent completion of the timer's callback
1681 * function. The timer's handler must not call add_timer_on(). Upon exit
1682 * the timer is not queued and the handler is not running on any CPU.
1685 * interrupt context. Even if the lock has nothing to do with the timer in
1703 * This function cannot guarantee that the timer is not rearmed again by
1712 * * %0 - The timer was not pending
1713 * * %1 - The timer was pending and deactivated
1715 int timer_delete_sync(struct timer_list *timer)
1717 return __timer_delete_sync(timer, false);
1722 * timer_shutdown_sync - Shutdown a timer and prevent rearming
1723 * @timer: The timer to be shutdown
1726 * - @timer is not queued
1727 * - The callback function of @timer is not running
1728 * - @timer cannot be enqueued again. Any attempt to rearm
1729 * @timer is silently ignored.
1734 * the timer is subject to a circular dependency problem.
1736 * A common pattern for this is a timer and a workqueue where the timer can
1737 * schedule work and work can arm the timer. On shutdown the workqueue must
1738 * be destroyed and the timer must be prevented from rearming. Unless the
1745 * timer_shutdown_sync(&mything->timer);
1750 * This obviously implies that the timer is not required to be functional
1754 * * %0 - The timer was not pending
1755 * * %1 - The timer was pending
1757 int timer_shutdown_sync(struct timer_list *timer)
1759 return __timer_delete_sync(timer, true);
1763 static void call_timer_fn(struct timer_list *timer,
1771 * It is permissible to free the timer from inside the
1775 * timer->lockdep_map, make a copy and use that here.
1779 lockdep_copy_map(&lockdep_map, &timer->lockdep_map);
1788 trace_timer_expire_entry(timer, baseclk);
1789 fn(timer);
1790 trace_timer_expire_exit(timer);
1795 WARN_ONCE(1, "timer: %pS preempt leak: %08x -> %08x\n",
1817 struct timer_list *timer;
1820 timer = hlist_entry(head->first, struct timer_list, entry);
1822 base->running_timer = timer;
1823 detach_timer(timer, true);
1825 fn = timer->function;
1833 if (timer->flags & TIMER_IRQSAFE) {
1835 call_timer_fn(timer, fn, baseclk);
1840 call_timer_fn(timer, fn, baseclk);
1893 * Search the first expiring timer in the various clock levels. Caller must
1971 * Check, if the next hrtimer event is before the next timer wheel
1986 * If the next timer is already expired, return the tick base
1995 * make sure that this tick really expires the timer to avoid
2011 * unnecessary raise of the timer softirq when the next_expiry value
2012 * will be reached even if there is no timer pending.
2015 * easy comparable to find out which base holds the first pending timer.
2056 * timer, it is possible that it will be missed completely.
2089 * Stores the next pending local and global timer expiry values in the
2094 * Caller needs to make sure timer base locks are held (use
2116 * timer_unlock_remote_bases - unlock timer bases of cpu
2119 * Unlocks the remote timer bases.
2135 * timer_lock_remote_bases - lock timer bases of cpu
2138 * Locks the remote timer bases.
2156 * timer_base_is_idle() - Return whether timer base is set idle
2158 * Returns value of local timer base is_idle value.
2194 * If the CPU is the last going idle in timer migration hierarchy, make
2218 * timer on !SMP systems.
2253 * timer migration hierarchy related functions. The value for the next
2254 * global timer in @tevt struct equals then KTIME_MAX. This is also
2255 * true, when the timer base is idle.
2257 * The proper timer migration hierarchy function depends on the callsite
2258 * and whether timer base is idle or not. @nextevt will be updated when
2259 * this CPU needs to handle the first timer migration hierarchy
2281 * global timer into timer migration hierarchy. Therefore a new
2284 * If the base is marked idle then any timer add operation must
2304 * When timer base is not set idle, undo the effect of
2306 * timer base but inactive timer migration hierarchy.
2308 * When timer base was already marked idle, nothing will be
2322 * get_next_timer_interrupt() - return the time (clock mono) of the next timer
2326 * Returns the tick aligned clock monotonic time of the next pending timer or
2327 * KTIME_MAX if no timer is pending. If timer of global base was queued into
2328 * timer migration hierarchy, first global timer is not taken into account. If
2329 * it was the last CPU of timer migration hierarchy going idle, first global
2338 * timer_base_try_to_set_idle() - Try to set the idle state of the timer bases
2344 * Returns the tick aligned clock monotonic time of the next pending timer or
2345 * KTIME_MAX if no timer is pending. When tick was already stopped KTIME_MAX is
2357 * timer_clear_idle - Clear the idle state of the timer base
2364 * We do this unlocked. The worst outcome is a remote pinned timer
2382 * @base: the timer vector to be processed.
2399 * timer at this clk are that all matching timers have been
2400 * dequeued or no timer has been queued since
2439 * This function runs timers and the timer-tq in bottom half context.
2454 * Called by the local, per-CPU timer interrupt on SMP.
2477 * worst outcome is a superfluous raise of the timer softirq
2480 * 2. A new first pinned timer is enqueued by a remote CPU
2485 * is missed, then the timer would expire one jiffy late -
2505 * Called from the timer interrupt handler to charge one tick to the current
2512 /* Note: this timer irq context must be accounted for as well. */
2528 struct timer_list *timer;
2532 timer = hlist_entry(head->first, struct timer_list, entry);
2533 detach_timer(timer, false);
2534 timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;
2535 internal_add_timer(new_base, timer);