184 static inline int tick_sched_flag_test(struct tick_sched *ts,  in tick_sched_flag_test()  argument
187 	return !!(ts->flags & flag);  in tick_sched_flag_test()
190 static inline void tick_sched_flag_set(struct tick_sched *ts,  in tick_sched_flag_set()  argument
194 	ts->flags |= flag;  in tick_sched_flag_set()
197 static inline void tick_sched_flag_clear(struct tick_sched *ts,  in tick_sched_flag_clear()  argument
201 	ts->flags &= ~flag;  in tick_sched_flag_clear()
206 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)  in tick_sched_do_timer()  argument
238 	if (ts->last_tick_jiffies != jiffies) {  in tick_sched_do_timer()
239 		ts->stalled_jiffies = 0;  in tick_sched_do_timer()
240 		ts->last_tick_jiffies = READ_ONCE(jiffies);  in tick_sched_do_timer()
242 		if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) {  in tick_sched_do_timer()
244 			ts->stalled_jiffies = 0;  in tick_sched_do_timer()
245 			ts->last_tick_jiffies = READ_ONCE(jiffies);  in tick_sched_do_timer()
249 	if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))  in tick_sched_do_timer()
250 		ts->got_idle_tick = 1;  in tick_sched_do_timer()
253 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)  in tick_sched_handle()  argument
264 	    tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {  in tick_sched_handle()
267 			ts->idle_jiffies++;  in tick_sched_handle()
273 		ts->next_tick = 0;  in tick_sched_handle()
286 	struct tick_sched *ts =	container_of(timer, struct tick_sched, sched_timer);  in tick_nohz_handler()  local
290 	tick_sched_do_timer(ts, now);  in tick_nohz_handler()
297 		tick_sched_handle(ts, regs);  in tick_nohz_handler()
299 		ts->next_tick = 0;  in tick_nohz_handler()
306 	if (unlikely(tick_sched_flag_test(ts, TS_FLAG_STOPPED)))  in tick_nohz_handler()
358 static bool can_stop_full_tick(int cpu, struct tick_sched *ts)  in can_stop_full_tick()  argument
368 	if (check_tick_dependency(&ts->tick_dep_mask))  in can_stop_full_tick()
508 	struct tick_sched *ts;  in tick_nohz_dep_set_cpu()  local
510 	ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_dep_set_cpu()
512 	prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);  in tick_nohz_dep_set_cpu()
530 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_dep_clear_cpu()  local
532 	atomic_andnot(BIT(bit), &ts->tick_dep_mask);  in tick_nohz_dep_clear_cpu()
585 	struct tick_sched *ts;  in __tick_nohz_task_switch()  local
590 	ts = this_cpu_ptr(&tick_cpu_sched);  in __tick_nohz_task_switch()
592 	if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {  in __tick_nohz_task_switch()
687 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_tick_stopped()  local
689 	return tick_sched_flag_test(ts, TS_FLAG_STOPPED);  in tick_nohz_tick_stopped()
694 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_tick_stopped_cpu()  local
696 	return tick_sched_flag_test(ts, TS_FLAG_STOPPED);  in tick_nohz_tick_stopped_cpu()
723 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)  in tick_nohz_stop_idle()  argument
727 	if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)))  in tick_nohz_stop_idle()
730 	delta = ktime_sub(now, ts->idle_entrytime);  in tick_nohz_stop_idle()
732 	write_seqcount_begin(&ts->idle_sleeptime_seq);  in tick_nohz_stop_idle()
734 		ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);  in tick_nohz_stop_idle()
736 		ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);  in tick_nohz_stop_idle()
738 	ts->idle_entrytime = now;  in tick_nohz_stop_idle()
739 	tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE);  in tick_nohz_stop_idle()
740 	write_seqcount_end(&ts->idle_sleeptime_seq);  in tick_nohz_stop_idle()
745 static void tick_nohz_start_idle(struct tick_sched *ts)  in tick_nohz_start_idle()  argument
747 	write_seqcount_begin(&ts->idle_sleeptime_seq);  in tick_nohz_start_idle()
748 	ts->idle_entrytime = ktime_get();  in tick_nohz_start_idle()
749 	tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE);  in tick_nohz_start_idle()
750 	write_seqcount_end(&ts->idle_sleeptime_seq);  in tick_nohz_start_idle()
755 static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime,  in get_cpu_sleep_time_us()  argument
769 		seq = read_seqcount_begin(&ts->idle_sleeptime_seq);  in get_cpu_sleep_time_us()
771 		if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) {  in get_cpu_sleep_time_us()
772 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_sleep_time_us()
778 	} while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq));  in get_cpu_sleep_time_us()
803 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_idle_time_us()  local
805 	return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime,  in get_cpu_idle_time_us()
829 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_iowait_time_us()  local
831 	return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime,  in get_cpu_iowait_time_us()
836 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart()  argument
838 	hrtimer_cancel(&ts->sched_timer);  in tick_nohz_restart()
839 	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);  in tick_nohz_restart()
842 	hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);  in tick_nohz_restart()
844 	if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {  in tick_nohz_restart()
845 		hrtimer_start_expires(&ts->sched_timer,  in tick_nohz_restart()
848 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_restart()
855 	ts->next_tick = 0;  in tick_nohz_restart()
891 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)  in tick_nohz_next_event()  argument
898 	ts->last_jiffies = basejiff;  in tick_nohz_next_event()
899 	ts->timer_expires_base = basemono;  in tick_nohz_next_event()
923 		ts->next_timer = next_tick;  in tick_nohz_next_event()
940 		if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {  in tick_nohz_next_event()
941 			ts->timer_expires = 0;  in tick_nohz_next_event()
954 	    (tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST)))  in tick_nohz_next_event()
963 	ts->timer_expires = min_t(u64, expires, next_tick);  in tick_nohz_next_event()
966 	return ts->timer_expires;  in tick_nohz_next_event()
969 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)  in tick_nohz_stop_tick()  argument
972 	unsigned long basejiff = ts->last_jiffies;  in tick_nohz_stop_tick()
973 	u64 basemono = ts->timer_expires_base;  in tick_nohz_stop_tick()
974 	bool timer_idle = tick_sched_flag_test(ts, TS_FLAG_STOPPED);  in tick_nohz_stop_tick()
979 	ts->timer_expires_base = 0;  in tick_nohz_stop_tick()
986 	if (expires > ts->timer_expires) {  in tick_nohz_stop_tick()
998 		expires = ts->timer_expires;  in tick_nohz_stop_tick()
1016 		tick_sched_flag_set(ts, TS_FLAG_DO_TIMER_LAST);  in tick_nohz_stop_tick()
1018 		tick_sched_flag_clear(ts, TS_FLAG_DO_TIMER_LAST);  in tick_nohz_stop_tick()
1022 	if (tick_sched_flag_test(ts, TS_FLAG_STOPPED) && (expires == ts->next_tick)) {  in tick_nohz_stop_tick()
1024 		if (expires == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))  in tick_nohz_stop_tick()
1028 			  "timer->active: %d timer->expires: %llu\n", basemono, ts->next_tick,  in tick_nohz_stop_tick()
1029 			  dev->next_event, hrtimer_active(&ts->sched_timer),  in tick_nohz_stop_tick()
1030 			  hrtimer_get_expires(&ts->sched_timer));  in tick_nohz_stop_tick()
1040 	if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {  in tick_nohz_stop_tick()
1044 		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);  in tick_nohz_stop_tick()
1045 		tick_sched_flag_set(ts, TS_FLAG_STOPPED);  in tick_nohz_stop_tick()
1049 	ts->next_tick = expires;  in tick_nohz_stop_tick()
1056 		if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))  in tick_nohz_stop_tick()
1057 			hrtimer_cancel(&ts->sched_timer);  in tick_nohz_stop_tick()
1063 	if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {  in tick_nohz_stop_tick()
1064 		hrtimer_start(&ts->sched_timer, expires,  in tick_nohz_stop_tick()
1067 		hrtimer_set_expires(&ts->sched_timer, expires);  in tick_nohz_stop_tick()
1072 static void tick_nohz_retain_tick(struct tick_sched *ts)  in tick_nohz_retain_tick()  argument
1074 	ts->timer_expires_base = 0;  in tick_nohz_retain_tick()
1078 static void tick_nohz_full_stop_tick(struct tick_sched *ts, int cpu)  in tick_nohz_full_stop_tick()  argument
1080 	if (tick_nohz_next_event(ts, cpu))  in tick_nohz_full_stop_tick()
1081 		tick_nohz_stop_tick(ts, cpu);  in tick_nohz_full_stop_tick()
1083 		tick_nohz_retain_tick(ts);  in tick_nohz_full_stop_tick()
1087 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart_sched_tick()  argument
1102 	tick_sched_flag_clear(ts, TS_FLAG_STOPPED);  in tick_nohz_restart_sched_tick()
1103 	tick_nohz_restart(ts, now);  in tick_nohz_restart_sched_tick()
1106 static void __tick_nohz_full_update_tick(struct tick_sched *ts,  in __tick_nohz_full_update_tick()  argument
1112 	if (can_stop_full_tick(cpu, ts))  in __tick_nohz_full_update_tick()
1113 		tick_nohz_full_stop_tick(ts, cpu);  in __tick_nohz_full_update_tick()
1114 	else if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))  in __tick_nohz_full_update_tick()
1115 		tick_nohz_restart_sched_tick(ts, now);  in __tick_nohz_full_update_tick()
1119 static void tick_nohz_full_update_tick(struct tick_sched *ts)  in tick_nohz_full_update_tick()  argument
1124 	if (!tick_sched_flag_test(ts, TS_FLAG_NOHZ))  in tick_nohz_full_update_tick()
1127 	__tick_nohz_full_update_tick(ts, ktime_get());  in tick_nohz_full_update_tick()
1169 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)  in can_stop_idle_tick()  argument
1173 	if (unlikely(!tick_sched_flag_test(ts, TS_FLAG_NOHZ)))  in can_stop_idle_tick()
1207 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_stop_tick()  local
1215 	if (ts->timer_expires_base)  in tick_nohz_idle_stop_tick()
1216 		expires = ts->timer_expires;  in tick_nohz_idle_stop_tick()
1217 	else if (can_stop_idle_tick(cpu, ts))  in tick_nohz_idle_stop_tick()
1218 		expires = tick_nohz_next_event(ts, cpu);  in tick_nohz_idle_stop_tick()
1222 	ts->idle_calls++;  in tick_nohz_idle_stop_tick()
1225 		int was_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);  in tick_nohz_idle_stop_tick()
1227 		tick_nohz_stop_tick(ts, cpu);  in tick_nohz_idle_stop_tick()
1229 		ts->idle_sleeps++;  in tick_nohz_idle_stop_tick()
1230 		ts->idle_expires = expires;  in tick_nohz_idle_stop_tick()
1232 		if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {  in tick_nohz_idle_stop_tick()
1233 			ts->idle_jiffies = ts->last_jiffies;  in tick_nohz_idle_stop_tick()
1237 		tick_nohz_retain_tick(ts);  in tick_nohz_idle_stop_tick()
1253 	struct tick_sched *ts;  in tick_nohz_idle_enter()  local
1259 	ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_enter()
1261 	WARN_ON_ONCE(ts->timer_expires_base);  in tick_nohz_idle_enter()
1263 	tick_sched_flag_set(ts, TS_FLAG_INIDLE);  in tick_nohz_idle_enter()
1264 	tick_nohz_start_idle(ts);  in tick_nohz_idle_enter()
1290 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_exit()  local
1292 	if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))  in tick_nohz_irq_exit()
1293 		tick_nohz_start_idle(ts);  in tick_nohz_irq_exit()
1295 		tick_nohz_full_update_tick(ts);  in tick_nohz_irq_exit()
1305 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_got_tick()  local
1307 	if (ts->got_idle_tick) {  in tick_nohz_idle_got_tick()
1308 		ts->got_idle_tick = 0;  in tick_nohz_idle_got_tick()
1343 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_get_sleep_length()  local
1349 	ktime_t now = ts->idle_entrytime;  in tick_nohz_get_sleep_length()
1352 	WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));  in tick_nohz_get_sleep_length()
1356 	if (!can_stop_idle_tick(cpu, ts))  in tick_nohz_get_sleep_length()
1359 	next_event = tick_nohz_next_event(ts, cpu);  in tick_nohz_get_sleep_length()
1368 			   hrtimer_next_event_without(&ts->sched_timer));  in tick_nohz_get_sleep_length()
1384 	struct tick_sched *ts = tick_get_tick_sched(cpu);  in tick_nohz_get_idle_calls_cpu()  local
1386 	return ts->idle_calls;  in tick_nohz_get_idle_calls_cpu()
1389 static void tick_nohz_account_idle_time(struct tick_sched *ts,  in tick_nohz_account_idle_time()  argument
1394 	ts->idle_exittime = now;  in tick_nohz_account_idle_time()
1403 	ticks = jiffies - ts->idle_jiffies;  in tick_nohz_account_idle_time()
1413 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_restart_tick()  local
1415 	if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {  in tick_nohz_idle_restart_tick()
1417 		tick_nohz_restart_sched_tick(ts, now);  in tick_nohz_idle_restart_tick()
1418 		tick_nohz_account_idle_time(ts, now);  in tick_nohz_idle_restart_tick()
1422 static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_idle_update_tick()  argument
1425 		__tick_nohz_full_update_tick(ts, now);  in tick_nohz_idle_update_tick()
1427 		tick_nohz_restart_sched_tick(ts, now);  in tick_nohz_idle_update_tick()
1429 	tick_nohz_account_idle_time(ts, now);  in tick_nohz_idle_update_tick()
1450 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_exit()  local
1456 	WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));  in tick_nohz_idle_exit()
1457 	WARN_ON_ONCE(ts->timer_expires_base);  in tick_nohz_idle_exit()
1459 	tick_sched_flag_clear(ts, TS_FLAG_INIDLE);  in tick_nohz_idle_exit()
1460 	idle_active = tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE);  in tick_nohz_idle_exit()
1461 	tick_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);  in tick_nohz_idle_exit()
1467 		tick_nohz_stop_idle(ts, now);  in tick_nohz_idle_exit()
1470 		tick_nohz_idle_update_tick(ts, now);  in tick_nohz_idle_exit()
1483 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_lowres_handler()  local
1487 	if (likely(tick_nohz_handler(&ts->sched_timer) == HRTIMER_RESTART))  in tick_nohz_lowres_handler()
1488 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_lowres_handler()
1491 static inline void tick_nohz_activate(struct tick_sched *ts)  in tick_nohz_activate()  argument
1495 	tick_sched_flag_set(ts, TS_FLAG_NOHZ);  in tick_nohz_activate()
1521 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_enter()  local
1524 	if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE))  in tick_nohz_irq_enter()
1527 	if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))  in tick_nohz_irq_enter()
1528 		tick_nohz_stop_idle(ts, now);  in tick_nohz_irq_enter()
1536 	if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))  in tick_nohz_irq_enter()
1544 static inline void tick_nohz_activate(struct tick_sched *ts) { }  in tick_nohz_activate()  argument
1573 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_setup_sched_timer()  local
1576 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);  in tick_setup_sched_timer()
1579 		tick_sched_flag_set(ts, TS_FLAG_HIGHRES);  in tick_setup_sched_timer()
1580 		ts->sched_timer.function = tick_nohz_handler;  in tick_setup_sched_timer()
1584 	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());  in tick_setup_sched_timer()
1591 		hrtimer_add_expires_ns(&ts->sched_timer, offset);  in tick_setup_sched_timer()
1594 	hrtimer_forward_now(&ts->sched_timer, TICK_NSEC);  in tick_setup_sched_timer()
1596 		hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);  in tick_setup_sched_timer()
1598 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_setup_sched_timer()
1599 	tick_nohz_activate(ts);  in tick_setup_sched_timer()
1608 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_sched_timer_dying()  local
1613 	if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))  in tick_sched_timer_dying()
1614 		hrtimer_cancel(&ts->sched_timer);  in tick_sched_timer_dying()
1616 	idle_sleeptime = ts->idle_sleeptime;  in tick_sched_timer_dying()
1617 	iowait_sleeptime = ts->iowait_sleeptime;  in tick_sched_timer_dying()
1618 	idle_calls = ts->idle_calls;  in tick_sched_timer_dying()
1619 	idle_sleeps = ts->idle_sleeps;  in tick_sched_timer_dying()
1620 	memset(ts, 0, sizeof(*ts));  in tick_sched_timer_dying()
1621 	ts->idle_sleeptime = idle_sleeptime;  in tick_sched_timer_dying()
1622 	ts->iowait_sleeptime = iowait_sleeptime;  in tick_sched_timer_dying()
1623 	ts->idle_calls = idle_calls;  in tick_sched_timer_dying()
1624 	ts->idle_sleeps = idle_sleeps;  in tick_sched_timer_dying()
1643 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_oneshot_notify()  local
1645 	set_bit(0, &ts->check_clocks);  in tick_oneshot_notify()
1658 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_check_oneshot_change()  local
1660 	if (!test_and_clear_bit(0, &ts->check_clocks))  in tick_check_oneshot_change()
1663 	if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))  in tick_check_oneshot_change()