Lines Matching full:cs

23 static void clocksource_enqueue(struct clocksource *cs);
25 static noinline u64 cycles_to_nsec_safe(struct clocksource *cs, u64 start, u64 end) in cycles_to_nsec_safe() argument
27 u64 delta = clocksource_delta(end, start, cs->mask, cs->max_raw_delta); in cycles_to_nsec_safe()
29 if (likely(delta < cs->max_cycles)) in cycles_to_nsec_safe()
30 return clocksource_cyc2ns(delta, cs->mult, cs->shift); in cycles_to_nsec_safe()
32 return mul_u64_u32_shr(delta, cs->mult, cs->shift); in cycles_to_nsec_safe()
125 * a lower bound for cs->uncertainty_margin values when registering clocks.
144 * Default for maximum permissible skew when cs->uncertainty_margin is
145 * not specified, and the lower bound even when cs->uncertainty_margin
147 * clocks with unspecifed cs->uncertainty_margin, so this macro is used
195 static void clocksource_change_rating(struct clocksource *cs, int rating) in clocksource_change_rating() argument
197 list_del(&cs->list); in clocksource_change_rating()
198 cs->rating = rating; in clocksource_change_rating()
199 clocksource_enqueue(cs); in clocksource_change_rating()
202 static void __clocksource_unstable(struct clocksource *cs) in __clocksource_unstable() argument
204 cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG); in __clocksource_unstable()
205 cs->flags |= CLOCK_SOURCE_UNSTABLE; in __clocksource_unstable()
211 if (list_empty(&cs->list)) { in __clocksource_unstable()
212 cs->rating = 0; in __clocksource_unstable()
216 if (cs->mark_unstable) in __clocksource_unstable()
217 cs->mark_unstable(cs); in __clocksource_unstable()
226 * @cs: clocksource to be marked unstable
231 void clocksource_mark_unstable(struct clocksource *cs) in clocksource_mark_unstable() argument
236 if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) { in clocksource_mark_unstable()
237 if (!list_empty(&cs->list) && list_empty(&cs->wd_list)) in clocksource_mark_unstable()
238 list_add(&cs->wd_list, &watchdog_list); in clocksource_mark_unstable()
239 __clocksource_unstable(cs); in clocksource_mark_unstable()
253 static enum wd_read_status cs_watchdog_read(struct clocksource *cs, u64 *csnow, u64 *wdnow) in cs_watchdog_read() argument
264 *csnow = cs->read(cs); in cs_watchdog_read()
270 if (wd_delay <= md + cs->uncertainty_margin) { in cs_watchdog_read()
293 smp_processor_id(), cs->name, wd_delay, WATCHDOG_MAX_SKEW, wd_seq_delay, nretries, cs->name); in cs_watchdog_read()
300 cs->name, wd_delay); in cs_watchdog_read()
359 struct clocksource *cs = (struct clocksource *)csin; in clocksource_verify_one_cpu() local
361 csnow_mid = cs->read(cs); in clocksource_verify_one_cpu()
364 void clocksource_verify_percpu(struct clocksource *cs) in clocksource_verify_percpu() argument
381 pr_warn("Not enough CPUs to check clocksource '%s'.\n", cs->name); in clocksource_verify_percpu()
386 cs->name, testcpu, cpumask_pr_args(&cpus_chosen)); in clocksource_verify_percpu()
391 csnow_begin = cs->read(cs); in clocksource_verify_percpu()
392 smp_call_function_single(cpu, clocksource_verify_one_cpu, cs, 1); in clocksource_verify_percpu()
393 csnow_end = cs->read(cs); in clocksource_verify_percpu()
394 delta = (s64)((csnow_mid - csnow_begin) & cs->mask); in clocksource_verify_percpu()
397 delta = (csnow_end - csnow_mid) & cs->mask; in clocksource_verify_percpu()
400 cs_nsec = cycles_to_nsec_safe(cs, csnow_begin, csnow_end); in clocksource_verify_percpu()
411 cpumask_pr_args(&cpus_ahead), testcpu, cs->name); in clocksource_verify_percpu()
414 cpumask_pr_args(&cpus_behind), testcpu, cs->name); in clocksource_verify_percpu()
417 testcpu, cs_nsec_min, cs_nsec_max, cs->name); in clocksource_verify_percpu()
423 struct clocksource *cs; in clocksource_reset_watchdog() local
425 list_for_each_entry(cs, &watchdog_list, wd_list) in clocksource_reset_watchdog()
426 cs->flags &= ~CLOCK_SOURCE_WATCHDOG; in clocksource_reset_watchdog()
435 struct clocksource *cs; in clocksource_watchdog() local
446 list_for_each_entry(cs, &watchdog_list, wd_list) { in clocksource_watchdog()
449 if (cs->flags & CLOCK_SOURCE_UNSTABLE) { in clocksource_watchdog()
455 read_ret = cs_watchdog_read(cs, &csnow, &wdnow); in clocksource_watchdog()
459 __clocksource_unstable(cs); in clocksource_watchdog()
473 * cs->last could keep unchanged for 5 minutes, reset in clocksource_watchdog()
482 if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) || in clocksource_watchdog()
484 cs->flags |= CLOCK_SOURCE_WATCHDOG; in clocksource_watchdog()
485 cs->wd_last = wdnow; in clocksource_watchdog()
486 cs->cs_last = csnow; in clocksource_watchdog()
490 wd_nsec = cycles_to_nsec_safe(watchdog, cs->wd_last, wdnow); in clocksource_watchdog()
491 cs_nsec = cycles_to_nsec_safe(cs, cs->cs_last, csnow); in clocksource_watchdog()
492 wdlast = cs->wd_last; /* save these in case we print them */ in clocksource_watchdog()
493 cslast = cs->cs_last; in clocksource_watchdog()
494 cs->cs_last = csnow; in clocksource_watchdog()
495 cs->wd_last = wdnow; in clocksource_watchdog()
522 md = cs->uncertainty_margin + watchdog->uncertainty_margin; in clocksource_watchdog()
529 smp_processor_id(), cs->name); in clocksource_watchdog()
533 cs->name, cs_nsec, csnow, cslast, cs->mask); in clocksource_watchdog()
537 cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec); in clocksource_watchdog()
538 if (curr_clocksource == cs) in clocksource_watchdog()
539 pr_warn(" '%s' is current clocksource.\n", cs->name); in clocksource_watchdog()
541 … '%s' (not '%s') is current clocksource.\n", curr_clocksource->name, cs->name); in clocksource_watchdog()
544 __clocksource_unstable(cs); in clocksource_watchdog()
548 if (cs == curr_clocksource && cs->tick_stable) in clocksource_watchdog()
549 cs->tick_stable(cs); in clocksource_watchdog()
551 if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && in clocksource_watchdog()
552 (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) && in clocksource_watchdog()
555 cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES; in clocksource_watchdog()
572 if (cs != curr_clocksource) { in clocksource_watchdog()
573 cs->flags |= CLOCK_SOURCE_RESELECT; in clocksource_watchdog()
631 static void clocksource_enqueue_watchdog(struct clocksource *cs) in clocksource_enqueue_watchdog() argument
633 INIT_LIST_HEAD(&cs->wd_list); in clocksource_enqueue_watchdog()
635 if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) { in clocksource_enqueue_watchdog()
636 /* cs is a clocksource to be watched. */ in clocksource_enqueue_watchdog()
637 list_add(&cs->wd_list, &watchdog_list); in clocksource_enqueue_watchdog()
638 cs->flags &= ~CLOCK_SOURCE_WATCHDOG; in clocksource_enqueue_watchdog()
640 /* cs is a watchdog. */ in clocksource_enqueue_watchdog()
641 if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) in clocksource_enqueue_watchdog()
642 cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES; in clocksource_enqueue_watchdog()
648 struct clocksource *cs, *old_wd; in clocksource_select_watchdog() local
657 list_for_each_entry(cs, &clocksource_list, list) { in clocksource_select_watchdog()
658 /* cs is a clocksource to be watched. */ in clocksource_select_watchdog()
659 if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) in clocksource_select_watchdog()
663 if (fallback && cs == old_wd) in clocksource_select_watchdog()
667 if (!watchdog || cs->rating > watchdog->rating) in clocksource_select_watchdog()
668 watchdog = cs; in clocksource_select_watchdog()
683 static void clocksource_dequeue_watchdog(struct clocksource *cs) in clocksource_dequeue_watchdog() argument
685 if (cs != watchdog) { in clocksource_dequeue_watchdog()
686 if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) { in clocksource_dequeue_watchdog()
687 /* cs is a watched clocksource. */ in clocksource_dequeue_watchdog()
688 list_del_init(&cs->wd_list); in clocksource_dequeue_watchdog()
697 struct clocksource *cs, *tmp; in __clocksource_watchdog_kthread() local
708 list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) { in __clocksource_watchdog_kthread()
709 if (cs->flags & CLOCK_SOURCE_UNSTABLE) { in __clocksource_watchdog_kthread()
710 list_del_init(&cs->wd_list); in __clocksource_watchdog_kthread()
711 clocksource_change_rating(cs, 0); in __clocksource_watchdog_kthread()
714 if (cs->flags & CLOCK_SOURCE_RESELECT) { in __clocksource_watchdog_kthread()
715 cs->flags &= ~CLOCK_SOURCE_RESELECT; in __clocksource_watchdog_kthread()
735 static bool clocksource_is_watchdog(struct clocksource *cs) in clocksource_is_watchdog() argument
737 return cs == watchdog; in clocksource_is_watchdog()
742 static void clocksource_enqueue_watchdog(struct clocksource *cs) in clocksource_enqueue_watchdog() argument
744 if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) in clocksource_enqueue_watchdog()
745 cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES; in clocksource_enqueue_watchdog()
749 static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { } in clocksource_dequeue_watchdog() argument
752 static bool clocksource_is_watchdog(struct clocksource *cs) { return false; } in clocksource_is_watchdog() argument
753 void clocksource_mark_unstable(struct clocksource *cs) { } in clocksource_mark_unstable() argument
760 static bool clocksource_is_suspend(struct clocksource *cs) in clocksource_is_suspend() argument
762 return cs == suspend_clocksource; in clocksource_is_suspend()
765 static void __clocksource_suspend_select(struct clocksource *cs) in __clocksource_suspend_select() argument
770 if (!(cs->flags & CLOCK_SOURCE_SUSPEND_NONSTOP)) in __clocksource_suspend_select()
778 if (cs->suspend || cs->resume) { in __clocksource_suspend_select()
780 cs->name); in __clocksource_suspend_select()
784 if (!suspend_clocksource || cs->rating > suspend_clocksource->rating) in __clocksource_suspend_select()
785 suspend_clocksource = cs; in __clocksource_suspend_select()
794 struct clocksource *cs, *old_suspend; in clocksource_suspend_select() local
800 list_for_each_entry(cs, &clocksource_list, list) { in clocksource_suspend_select()
802 if (fallback && cs == old_suspend) in clocksource_suspend_select()
805 __clocksource_suspend_select(cs); in clocksource_suspend_select()
811 * @cs: current clocksource from timekeeping
822 void clocksource_start_suspend_timing(struct clocksource *cs, u64 start_cycles) in clocksource_start_suspend_timing() argument
832 if (clocksource_is_suspend(cs)) { in clocksource_start_suspend_timing()
848 * @cs: current clocksource from timekeeping
860 u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now) in clocksource_stop_suspend_timing() argument
872 if (clocksource_is_suspend(cs)) in clocksource_stop_suspend_timing()
884 if (!clocksource_is_suspend(cs) && suspend_clocksource->disable) in clocksource_stop_suspend_timing()
895 struct clocksource *cs; in clocksource_suspend() local
897 list_for_each_entry_reverse(cs, &clocksource_list, list) in clocksource_suspend()
898 if (cs->suspend) in clocksource_suspend()
899 cs->suspend(cs); in clocksource_suspend()
907 struct clocksource *cs; in clocksource_resume() local
909 list_for_each_entry(cs, &clocksource_list, list) in clocksource_resume()
910 if (cs->resume) in clocksource_resume()
911 cs->resume(cs); in clocksource_resume()
930 * @cs: Pointer to clocksource
933 static u32 clocksource_max_adjustment(struct clocksource *cs) in clocksource_max_adjustment() argument
939 ret = (u64)cs->mult * 11; in clocksource_max_adjustment()
991 * @cs: Pointer to clocksource to be updated
994 static inline void clocksource_update_max_deferment(struct clocksource *cs) in clocksource_update_max_deferment() argument
996 cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift, in clocksource_update_max_deferment()
997 cs->maxadj, cs->mask, in clocksource_update_max_deferment()
998 &cs->max_cycles); in clocksource_update_max_deferment()
1007 cs->max_raw_delta = (cs->mask >> 1) + (cs->mask >> 2) + (cs->mask >> 3); in clocksource_update_max_deferment()
1012 struct clocksource *cs; in clocksource_find_best() local
1022 list_for_each_entry(cs, &clocksource_list, list) { in clocksource_find_best()
1023 if (skipcur && cs == curr_clocksource) in clocksource_find_best()
1025 if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES)) in clocksource_find_best()
1027 return cs; in clocksource_find_best()
1035 struct clocksource *best, *cs; in __clocksource_select() local
1046 list_for_each_entry(cs, &clocksource_list, list) { in __clocksource_select()
1047 if (skipcur && cs == curr_clocksource) in __clocksource_select()
1049 if (strcmp(cs->name, override_name) != 0) in __clocksource_select()
1056 if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) { in __clocksource_select()
1058 if (cs->flags & CLOCK_SOURCE_UNSTABLE) { in __clocksource_select()
1060 cs->name); in __clocksource_select()
1068 cs->name); in __clocksource_select()
1072 best = cs; in __clocksource_select()
1126 static void clocksource_enqueue(struct clocksource *cs) in clocksource_enqueue() argument
1133 if (tmp->rating < cs->rating) in clocksource_enqueue()
1137 list_add(&cs->list, entry); in clocksource_enqueue()
1142 * @cs: clocksource to be registered
1152 void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq) in __clocksource_update_freq_scale() argument
1170 sec = cs->mask; in __clocksource_update_freq_scale()
1175 else if (sec > 600 && cs->mask > UINT_MAX) in __clocksource_update_freq_scale()
1178 clocks_calc_mult_shift(&cs->mult, &cs->shift, freq, in __clocksource_update_freq_scale()
1197 if (scale && freq && !cs->uncertainty_margin) { in __clocksource_update_freq_scale()
1198 cs->uncertainty_margin = NSEC_PER_SEC / (scale * freq); in __clocksource_update_freq_scale()
1199 if (cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW) in __clocksource_update_freq_scale()
1200 cs->uncertainty_margin = 2 * WATCHDOG_MAX_SKEW; in __clocksource_update_freq_scale()
1201 } else if (!cs->uncertainty_margin) { in __clocksource_update_freq_scale()
1202 cs->uncertainty_margin = WATCHDOG_THRESHOLD; in __clocksource_update_freq_scale()
1204 WARN_ON_ONCE(cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW); in __clocksource_update_freq_scale()
1210 cs->maxadj = clocksource_max_adjustment(cs); in __clocksource_update_freq_scale()
1211 while (freq && ((cs->mult + cs->maxadj < cs->mult) in __clocksource_update_freq_scale()
1212 || (cs->mult - cs->maxadj > cs->mult))) { in __clocksource_update_freq_scale()
1213 cs->mult >>= 1; in __clocksource_update_freq_scale()
1214 cs->shift--; in __clocksource_update_freq_scale()
1215 cs->maxadj = clocksource_max_adjustment(cs); in __clocksource_update_freq_scale()
1222 WARN_ONCE(cs->mult + cs->maxadj < cs->mult, in __clocksource_update_freq_scale()
1224 cs->name); in __clocksource_update_freq_scale()
1226 clocksource_update_max_deferment(cs); in __clocksource_update_freq_scale()
1229 cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns); in __clocksource_update_freq_scale()
1235 * @cs: clocksource to be registered
1244 int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq) in __clocksource_register_scale() argument
1248 clocksource_arch_init(cs); in __clocksource_register_scale()
1250 if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX)) in __clocksource_register_scale()
1251 cs->id = CSID_GENERIC; in __clocksource_register_scale()
1252 if (cs->vdso_clock_mode < 0 || in __clocksource_register_scale()
1253 cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) { in __clocksource_register_scale()
1255 cs->name, cs->vdso_clock_mode); in __clocksource_register_scale()
1256 cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; in __clocksource_register_scale()
1260 __clocksource_update_freq_scale(cs, scale, freq); in __clocksource_register_scale()
1266 clocksource_enqueue(cs); in __clocksource_register_scale()
1267 clocksource_enqueue_watchdog(cs); in __clocksource_register_scale()
1272 __clocksource_suspend_select(cs); in __clocksource_register_scale()
1279 * Unbind clocksource @cs. Called with clocksource_mutex held
1281 static int clocksource_unbind(struct clocksource *cs) in clocksource_unbind() argument
1285 if (clocksource_is_watchdog(cs)) { in clocksource_unbind()
1288 if (clocksource_is_watchdog(cs)) in clocksource_unbind()
1292 if (cs == curr_clocksource) { in clocksource_unbind()
1295 if (curr_clocksource == cs) in clocksource_unbind()
1299 if (clocksource_is_suspend(cs)) { in clocksource_unbind()
1309 clocksource_dequeue_watchdog(cs); in clocksource_unbind()
1310 list_del_init(&cs->list); in clocksource_unbind()
1318 * @cs: clocksource to be unregistered
1320 int clocksource_unregister(struct clocksource *cs) in clocksource_unregister() argument
1325 if (!list_empty(&cs->list)) in clocksource_unregister()
1326 ret = clocksource_unbind(cs); in clocksource_unregister()
1412 struct clocksource *cs; in unbind_clocksource_store() local
1422 list_for_each_entry(cs, &clocksource_list, list) { in unbind_clocksource_store()
1423 if (strcmp(cs->name, name)) in unbind_clocksource_store()
1425 ret = clocksource_unbind(cs); in unbind_clocksource_store()