736 	struct scx_sched *sch = scx_root;  in __scx_update_idle()  local
771 	if (SCX_HAS_OP(sch, update_idle) && do_notify && !scx_rq_bypassing(rq))  in __scx_update_idle()
772 		SCX_CALL_OP(sch, SCX_KF_REST, update_idle, rq, cpu_of(rq), idle);  in __scx_update_idle()
822 static int validate_node(struct scx_sched *sch, int node)  in validate_node()  argument
825 		scx_error(sch, "per-node idle tracking is disabled");  in validate_node()
835 		scx_error(sch, "invalid node %d", node);  in validate_node()
841 		scx_error(sch, "unavailable node %d", node);  in validate_node()
850 static bool check_builtin_idle_enabled(struct scx_sched *sch)  in check_builtin_idle_enabled()  argument
855 	scx_error(sch, "built-in idle tracking is disabled");  in check_builtin_idle_enabled()
885 static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,  in select_cpu_from_kfunc()  argument
893 	if (!ops_cpu_valid(sch, prev_cpu, NULL))  in select_cpu_from_kfunc()
896 	if (!check_builtin_idle_enabled(sch))  in select_cpu_from_kfunc()
909 		if (!scx_kf_allowed(sch, SCX_KF_SELECT_CPU | SCX_KF_ENQUEUE))  in select_cpu_from_kfunc()
952 	struct scx_sched *sch;  in scx_bpf_cpu_node()  local
956 	sch = rcu_dereference(scx_root);  in scx_bpf_cpu_node()
957 	if (unlikely(!sch) || !ops_cpu_valid(sch, cpu, NULL))  in scx_bpf_cpu_node()
980 	struct scx_sched *sch;  in scx_bpf_select_cpu_dfl()  local
985 	sch = rcu_dereference(scx_root);  in scx_bpf_select_cpu_dfl()
986 	if (unlikely(!sch))  in scx_bpf_select_cpu_dfl()
989 	cpu = select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, NULL, 0);  in scx_bpf_select_cpu_dfl()
1033 	struct scx_sched *sch;  in __scx_bpf_select_cpu_and()  local
1037 	sch = rcu_dereference(scx_root);  in __scx_bpf_select_cpu_and()
1038 	if (unlikely(!sch))  in __scx_bpf_select_cpu_and()
1041 	return select_cpu_from_kfunc(sch, p, args->prev_cpu, args->wake_flags,  in __scx_bpf_select_cpu_and()
1051 	struct scx_sched *sch;  in scx_bpf_select_cpu_and()  local
1055 	sch = rcu_dereference(scx_root);  in scx_bpf_select_cpu_and()
1056 	if (unlikely(!sch))  in scx_bpf_select_cpu_and()
1059 	return select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags,  in scx_bpf_select_cpu_and()
1074 	struct scx_sched *sch;  in scx_bpf_get_idle_cpumask_node()  local
1078 	sch = rcu_dereference(scx_root);  in scx_bpf_get_idle_cpumask_node()
1079 	if (unlikely(!sch))  in scx_bpf_get_idle_cpumask_node()
1082 	node = validate_node(sch, node);  in scx_bpf_get_idle_cpumask_node()
1098 	struct scx_sched *sch;  in scx_bpf_get_idle_cpumask()  local
1102 	sch = rcu_dereference(scx_root);  in scx_bpf_get_idle_cpumask()
1103 	if (unlikely(!sch))  in scx_bpf_get_idle_cpumask()
1107 		scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled");  in scx_bpf_get_idle_cpumask()
1111 	if (!check_builtin_idle_enabled(sch))  in scx_bpf_get_idle_cpumask()
1129 	struct scx_sched *sch;  in scx_bpf_get_idle_smtmask_node()  local
1133 	sch = rcu_dereference(scx_root);  in scx_bpf_get_idle_smtmask_node()
1134 	if (unlikely(!sch))  in scx_bpf_get_idle_smtmask_node()
1137 	node = validate_node(sch, node);  in scx_bpf_get_idle_smtmask_node()
1157 	struct scx_sched *sch;  in scx_bpf_get_idle_smtmask()  local
1161 	sch = rcu_dereference(scx_root);  in scx_bpf_get_idle_smtmask()
1162 	if (unlikely(!sch))  in scx_bpf_get_idle_smtmask()
1166 		scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE enabled");  in scx_bpf_get_idle_smtmask()
1170 	if (!check_builtin_idle_enabled(sch))  in scx_bpf_get_idle_smtmask()
1206 	struct scx_sched *sch;  in scx_bpf_test_and_clear_cpu_idle()  local
1210 	sch = rcu_dereference(scx_root);  in scx_bpf_test_and_clear_cpu_idle()
1211 	if (unlikely(!sch))  in scx_bpf_test_and_clear_cpu_idle()
1214 	if (!check_builtin_idle_enabled(sch))  in scx_bpf_test_and_clear_cpu_idle()
1217 	if (!ops_cpu_valid(sch, cpu, NULL))  in scx_bpf_test_and_clear_cpu_idle()
1245 	struct scx_sched *sch;  in scx_bpf_pick_idle_cpu_node()  local
1249 	sch = rcu_dereference(scx_root);  in scx_bpf_pick_idle_cpu_node()
1250 	if (unlikely(!sch))  in scx_bpf_pick_idle_cpu_node()
1253 	node = validate_node(sch, node);  in scx_bpf_pick_idle_cpu_node()
1285 	struct scx_sched *sch;  in scx_bpf_pick_idle_cpu()  local
1289 	sch = rcu_dereference(scx_root);  in scx_bpf_pick_idle_cpu()
1290 	if (unlikely(!sch))  in scx_bpf_pick_idle_cpu()
1294 		scx_error(sch, "per-node idle tracking is enabled");  in scx_bpf_pick_idle_cpu()
1298 	if (!check_builtin_idle_enabled(sch))  in scx_bpf_pick_idle_cpu()
1328 	struct scx_sched *sch;  in scx_bpf_pick_any_cpu_node()  local
1333 	sch = rcu_dereference(scx_root);  in scx_bpf_pick_any_cpu_node()
1334 	if (unlikely(!sch))  in scx_bpf_pick_any_cpu_node()
1337 	node = validate_node(sch, node);  in scx_bpf_pick_any_cpu_node()
1375 	struct scx_sched *sch;  in scx_bpf_pick_any_cpu()  local
1380 	sch = rcu_dereference(scx_root);  in scx_bpf_pick_any_cpu()
1381 	if (unlikely(!sch))  in scx_bpf_pick_any_cpu()
1385 		scx_error(sch, "per-node idle tracking is enabled");  in scx_bpf_pick_any_cpu()