Lines Matching full:cc
179 #define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr argument
180 #define cc_exec_last_func(cc, dir) cc->cc_exec_entity[dir].cc_last_func argument
181 #define cc_exec_last_arg(cc, dir) cc->cc_exec_entity[dir].cc_last_arg argument
182 #define cc_exec_next(cc) cc->cc_next argument
183 #define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel argument
184 #define cc_exec_waiting(cc, dir) cc->cc_exec_entity[dir].cc_waiting argument
186 #define cc_migration_func(cc, dir) cc->cc_exec_entity[dir].ce_migration_func argument
187 #define cc_migration_arg(cc, dir) cc->cc_exec_entity[dir].ce_migration_arg argument
188 #define cc_migration_cpu(cc, dir) cc->cc_exec_entity[dir].ce_migration_cpu argument
189 #define cc_migration_time(cc, dir) cc->cc_exec_entity[dir].ce_migration_time argument
190 #define cc_migration_prec(cc, dir) cc->cc_exec_entity[dir].ce_migration_prec argument
201 #define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock) argument
202 #define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock) argument
203 #define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED) argument
207 static void callout_cpu_init(struct callout_cpu *cc, int cpu);
208 static void softclock_call_cc(struct callout *c, struct callout_cpu *cc,
237 cc_cce_cleanup(struct callout_cpu *cc, int direct) in cc_cce_cleanup() argument
240 cc_exec_curr(cc, direct) = NULL; in cc_cce_cleanup()
241 cc_exec_cancel(cc, direct) = false; in cc_cce_cleanup()
242 cc_exec_waiting(cc, direct) = false; in cc_cce_cleanup()
244 cc_migration_cpu(cc, direct) = CPUBLOCK; in cc_cce_cleanup()
245 cc_migration_time(cc, direct) = 0; in cc_cce_cleanup()
246 cc_migration_prec(cc, direct) = 0; in cc_cce_cleanup()
247 cc_migration_func(cc, direct) = NULL; in cc_cce_cleanup()
248 cc_migration_arg(cc, direct) = NULL; in cc_cce_cleanup()
256 cc_cce_migrating(struct callout_cpu *cc, int direct) in cc_cce_migrating() argument
260 return (cc_migration_cpu(cc, direct) != CPUBLOCK); in cc_cce_migrating()
273 struct callout_cpu *cc; in callout_callwheel_init() local
304 cc = CC_CPU(cpu); in callout_callwheel_init()
305 callout_cpu_init(cc, cpu); in callout_callwheel_init()
314 callout_cpu_init(struct callout_cpu *cc, int cpu) in callout_cpu_init() argument
318 mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN); in callout_cpu_init()
319 cc->cc_callwheel = malloc_domainset(sizeof(struct callout_list) * in callout_cpu_init()
323 LIST_INIT(&cc->cc_callwheel[i]); in callout_cpu_init()
324 TAILQ_INIT(&cc->cc_expireq); in callout_cpu_init()
325 cc->cc_firstevent = SBT_MAX; in callout_cpu_init()
327 cc_cce_cleanup(cc, i); in callout_cpu_init()
329 snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name), in callout_cpu_init()
341 callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu) in callout_cpu_switch() argument
345 MPASS(c != NULL && cc != NULL); in callout_cpu_switch()
346 CC_LOCK_ASSERT(cc); in callout_cpu_switch()
355 CC_UNLOCK(cc); in callout_cpu_switch()
372 struct callout_cpu *cc; in start_softclock() local
378 cc = CC_CPU(cpu); in start_softclock()
379 error = kproc_kthread_add(softclock_thread, cc, &p, &td, in start_softclock()
384 CC_LOCK(cc); in start_softclock()
385 cc->cc_thread = td; in start_softclock()
390 thread_lock_set(td, (struct mtx *)&cc->cc_lock); in start_softclock()
427 struct callout_cpu *cc; in callout_process() member
432 struct callout_cpu *cc; in callout_process() local
441 cc = CC_SELF(); in callout_process()
442 mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); in callout_process()
445 firstb = callout_hash(cc->cc_lastscan); in callout_process()
446 cc->cc_lastscan = now; in callout_process()
475 sc = &cc->cc_callwheel[firstb & callwheelmask]; in callout_process()
487 cc_exec_next(cc) = next; in callout_process()
488 cc->cc_bucket = firstb & callwheelmask; in callout_process()
490 softclock_call_cc(c, cc, in callout_process()
495 next = cc_exec_next(cc); in callout_process()
496 cc_exec_next(cc) = NULL; in callout_process()
499 TAILQ_INSERT_TAIL(&cc->cc_expireq, in callout_process()
534 cc->cc_firstevent = last; in callout_process()
542 if (!TAILQ_EMPTY(&cc->cc_expireq)) { in callout_process()
543 entropy.cc = cc; in callout_process()
548 td = cc->cc_thread; in callout_process()
555 mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); in callout_process()
557 mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); in callout_process()
563 struct callout_cpu *cc; in callout_lock() local
575 cc = CC_CPU(cpu); in callout_lock()
576 CC_LOCK(cc); in callout_lock()
579 CC_UNLOCK(cc); in callout_lock()
581 return (cc); in callout_lock()
585 callout_cc_add(struct callout *c, struct callout_cpu *cc, in callout_cc_add() argument
591 CC_LOCK_ASSERT(cc); in callout_cc_add()
592 if (sbt < cc->cc_lastscan) in callout_cc_add()
593 sbt = cc->cc_lastscan; in callout_cc_add()
607 LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le); in callout_cc_add()
608 if (cc->cc_bucket == bucket) in callout_cc_add()
609 cc_exec_next(cc) = c; in callout_cc_add()
618 if (sbt < cc->cc_firstevent) { in callout_cc_add()
619 cc->cc_firstevent = sbt; in callout_cc_add()
625 softclock_call_cc(struct callout *c, struct callout_cpu *cc, in softclock_call_cc() argument
652 CC_LOCK_ASSERT(cc); in softclock_call_cc()
671 cc_exec_curr(cc, direct) = c; in softclock_call_cc()
672 cc_exec_last_func(cc, direct) = c_func; in softclock_call_cc()
673 cc_exec_last_arg(cc, direct) = c_arg; in softclock_call_cc()
674 cc_exec_cancel(cc, direct) = false; in softclock_call_cc()
679 cc_exec_curr(cc, direct) = NULL; in softclock_call_cc()
680 callout_cc_add(c, cc, in softclock_call_cc()
681 cc->cc_lastscan + c->c_precision / 2, in softclock_call_cc()
686 CC_UNLOCK(cc); in softclock_call_cc()
688 CC_UNLOCK(cc); in softclock_call_cc()
694 if (cc_exec_cancel(cc, direct)) { in softclock_call_cc()
700 cc_exec_cancel(cc, direct) = true; in softclock_call_cc()
715 CC_UNLOCK(cc); in softclock_call_cc()
722 KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running", in softclock_call_cc()
746 KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle"); in softclock_call_cc()
751 CC_LOCK(cc); in softclock_call_cc()
752 KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr")); in softclock_call_cc()
753 cc_exec_curr(cc, direct) = NULL; in softclock_call_cc()
754 if (cc_exec_waiting(cc, direct)) { in softclock_call_cc()
761 if (cc_cce_migrating(cc, direct)) { in softclock_call_cc()
762 cc_cce_cleanup(cc, direct); in softclock_call_cc()
770 cc_exec_waiting(cc, direct) = false; in softclock_call_cc()
771 CC_UNLOCK(cc); in softclock_call_cc()
772 wakeup(&cc_exec_waiting(cc, direct)); in softclock_call_cc()
773 CC_LOCK(cc); in softclock_call_cc()
774 } else if (cc_cce_migrating(cc, direct)) { in softclock_call_cc()
780 new_cpu = cc_migration_cpu(cc, direct); in softclock_call_cc()
781 new_time = cc_migration_time(cc, direct); in softclock_call_cc()
782 new_prec = cc_migration_prec(cc, direct); in softclock_call_cc()
783 new_func = cc_migration_func(cc, direct); in softclock_call_cc()
784 new_arg = cc_migration_arg(cc, direct); in softclock_call_cc()
785 cc_cce_cleanup(cc, direct); in softclock_call_cc()
801 new_cc = callout_cpu_switch(c, cc, new_cpu); in softclock_call_cc()
806 CC_LOCK(cc); in softclock_call_cc()
833 struct callout_cpu *cc; in softclock_thread() local
839 cc = (struct callout_cpu *)arg; in softclock_thread()
840 CC_LOCK(cc); in softclock_thread()
842 while (TAILQ_EMPTY(&cc->cc_expireq)) { in softclock_thread()
844 * Use CC_LOCK(cc) as the thread_lock while in softclock_thread()
848 thread_lock_set(td, (struct mtx *)&cc->cc_lock); in softclock_thread()
853 CC_LOCK(cc); in softclock_thread()
859 while ((c = TAILQ_FIRST(&cc->cc_expireq)) != NULL) { in softclock_thread()
860 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in softclock_thread()
861 softclock_call_cc(c, cc, in softclock_thread()
943 struct callout_cpu *cc; in callout_reset_sbt_on() local
963 cc = callout_lock(c); in callout_reset_sbt_on()
969 if (cc_exec_curr(cc, direct) == c) { in callout_reset_sbt_on()
975 if (c->c_lock != NULL && !cc_exec_cancel(cc, direct)) in callout_reset_sbt_on()
976 cancelled = cc_exec_cancel(cc, direct) = true; in callout_reset_sbt_on()
977 if (cc_exec_waiting(cc, direct)) { in callout_reset_sbt_on()
985 CC_UNLOCK(cc); in callout_reset_sbt_on()
997 cc_migration_cpu(cc, direct) = cpu; in callout_reset_sbt_on()
998 cc_migration_time(cc, direct) = to_sbt; in callout_reset_sbt_on()
999 cc_migration_prec(cc, direct) = precision; in callout_reset_sbt_on()
1000 cc_migration_func(cc, direct) = ftn; in callout_reset_sbt_on()
1001 cc_migration_arg(cc, direct) = arg; in callout_reset_sbt_on()
1003 CC_UNLOCK(cc); in callout_reset_sbt_on()
1010 if (cc_exec_next(cc) == c) in callout_reset_sbt_on()
1011 cc_exec_next(cc) = LIST_NEXT(c, c_links.le); in callout_reset_sbt_on()
1014 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in callout_reset_sbt_on()
1028 if (cc_exec_curr(cc, direct) == c) { in callout_reset_sbt_on()
1045 cc_migration_cpu(cc, direct) = cpu; in callout_reset_sbt_on()
1046 cc_migration_time(cc, direct) = to_sbt; in callout_reset_sbt_on()
1047 cc_migration_prec(cc, direct) = precision; in callout_reset_sbt_on()
1048 cc_migration_func(cc, direct) = ftn; in callout_reset_sbt_on()
1049 cc_migration_arg(cc, direct) = arg; in callout_reset_sbt_on()
1056 CC_UNLOCK(cc); in callout_reset_sbt_on()
1059 cc = callout_cpu_switch(c, cc, cpu); in callout_reset_sbt_on()
1063 callout_cc_add(c, cc, to_sbt, precision, ftn, arg, flags); in callout_reset_sbt_on()
1067 CC_UNLOCK(cc); in callout_reset_sbt_on()
1090 struct callout_cpu *cc, *old_cc; in _callout_stop_safe() local
1121 cc = callout_lock(c); in _callout_stop_safe()
1150 if (sq_locked != 0 && cc != old_cc) { in _callout_stop_safe()
1152 CC_UNLOCK(cc); in _callout_stop_safe()
1165 if (cc_exec_curr(cc, direct) == c) { in _callout_stop_safe()
1182 if (cc_exec_curr(cc, direct) == c) { in _callout_stop_safe()
1201 CC_UNLOCK(cc); in _callout_stop_safe()
1203 &cc_exec_waiting(cc, direct)); in _callout_stop_safe()
1205 old_cc = cc; in _callout_stop_safe()
1215 cc_exec_waiting(cc, direct) = true; in _callout_stop_safe()
1217 CC_UNLOCK(cc); in _callout_stop_safe()
1219 &cc_exec_waiting(cc, direct), in _callout_stop_safe()
1220 &cc->cc_lock.lock_object, "codrain", in _callout_stop_safe()
1223 &cc_exec_waiting(cc, direct), in _callout_stop_safe()
1233 } else if (use_lock && !cc_exec_cancel(cc, direct)) { in _callout_stop_safe()
1243 cc_exec_cancel(cc, direct) = true; in _callout_stop_safe()
1246 KASSERT(!cc_cce_migrating(cc, direct), in _callout_stop_safe()
1251 cc_migration_cpu(cc, direct) = CPUBLOCK; in _callout_stop_safe()
1252 cc_migration_time(cc, direct) = 0; in _callout_stop_safe()
1253 cc_migration_prec(cc, direct) = 0; in _callout_stop_safe()
1254 cc_migration_func(cc, direct) = NULL; in _callout_stop_safe()
1255 cc_migration_arg(cc, direct) = NULL; in _callout_stop_safe()
1258 CC_UNLOCK(cc); in _callout_stop_safe()
1279 cc_migration_cpu(cc, direct) = CPUBLOCK; in _callout_stop_safe()
1280 cc_migration_time(cc, direct) = 0; in _callout_stop_safe()
1281 cc_migration_prec(cc, direct) = 0; in _callout_stop_safe()
1282 cc_migration_func(cc, direct) = NULL; in _callout_stop_safe()
1283 cc_migration_arg(cc, direct) = NULL; in _callout_stop_safe()
1287 CC_UNLOCK(cc); in _callout_stop_safe()
1299 sleepq_release(&cc_exec_waiting(cc, direct)); in _callout_stop_safe()
1308 if (cc_exec_curr(cc, direct) != c) in _callout_stop_safe()
1310 CC_UNLOCK(cc); in _callout_stop_safe()
1321 if (cc_exec_next(cc) == c) in _callout_stop_safe()
1322 cc_exec_next(cc) = LIST_NEXT(c, c_links.le); in _callout_stop_safe()
1325 TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); in _callout_stop_safe()
1328 CC_UNLOCK(cc); in _callout_stop_safe()
1386 struct callout_cpu *cc; in sysctl_kern_callout_stat() local
1404 cc = CC_CPU(cpu); in sysctl_kern_callout_stat()
1405 CC_LOCK(cc); in sysctl_kern_callout_stat()
1407 sc = &cc->cc_callwheel[i]; in sysctl_kern_callout_stat()
1428 CC_UNLOCK(cc); in sysctl_kern_callout_stat()
1510 struct callout_cpu *cc; in _show_last_callout() local
1513 cc = CC_CPU(cpu); in _show_last_callout()
1514 func = cc_exec_last_func(cc, direct); in _show_last_callout()
1515 arg = cc_exec_last_arg(cc, direct); in _show_last_callout()