Lines Matching refs:cpu
36 cpumask_var_t cpu; member
72 static int scx_cpu_node_if_enabled(int cpu) in scx_cpu_node_if_enabled() argument
77 return cpu_to_node(cpu); in scx_cpu_node_if_enabled()
80 static bool scx_idle_test_and_clear_cpu(int cpu) in scx_idle_test_and_clear_cpu() argument
82 int node = scx_cpu_node_if_enabled(cpu); in scx_idle_test_and_clear_cpu()
83 struct cpumask *idle_cpus = idle_cpumask(node)->cpu; in scx_idle_test_and_clear_cpu()
91 const struct cpumask *smt = cpu_smt_mask(cpu); in scx_idle_test_and_clear_cpu()
106 else if (cpumask_test_cpu(cpu, idle_smts)) in scx_idle_test_and_clear_cpu()
107 __cpumask_clear_cpu(cpu, idle_smts); in scx_idle_test_and_clear_cpu()
110 return cpumask_test_and_clear_cpu(cpu, idle_cpus); in scx_idle_test_and_clear_cpu()
118 int cpu; in pick_idle_cpu_in_node() local
122 cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed); in pick_idle_cpu_in_node()
123 if (cpu < nr_cpu_ids) in pick_idle_cpu_in_node()
130 cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed); in pick_idle_cpu_in_node()
131 if (cpu >= nr_cpu_ids) in pick_idle_cpu_in_node()
135 if (scx_idle_test_and_clear_cpu(cpu)) in pick_idle_cpu_in_node()
136 return cpu; in pick_idle_cpu_in_node()
154 s32 cpu = -EBUSY; in pick_idle_cpu_from_online_nodes() local
182 cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); in pick_idle_cpu_from_online_nodes()
183 if (cpu >= 0) in pick_idle_cpu_from_online_nodes()
188 return cpu; in pick_idle_cpu_from_online_nodes()
203 s32 cpu; in scx_pick_idle_cpu() local
210 cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); in scx_pick_idle_cpu()
211 if (cpu >= 0) in scx_pick_idle_cpu()
212 return cpu; in scx_pick_idle_cpu()
232 static unsigned int llc_weight(s32 cpu) in llc_weight() argument
236 sd = rcu_dereference(per_cpu(sd_llc, cpu)); in llc_weight()
247 static struct cpumask *llc_span(s32 cpu) in llc_span() argument
251 sd = rcu_dereference(per_cpu(sd_llc, cpu)); in llc_span()
262 static unsigned int numa_weight(s32 cpu) in numa_weight() argument
267 sd = rcu_dereference(per_cpu(sd_numa, cpu)); in numa_weight()
281 static struct cpumask *numa_span(s32 cpu) in numa_span() argument
286 sd = rcu_dereference(per_cpu(sd_numa, cpu)); in numa_span()
302 int cpu; in llc_numa_mismatch() local
327 for_each_online_cpu(cpu) in llc_numa_mismatch()
328 if (llc_weight(cpu) != numa_weight(cpu)) in llc_numa_mismatch()
349 s32 cpu = cpumask_first(cpu_online_mask); in scx_idle_update_selcpu_topology() local
362 nr_cpus = llc_weight(cpu); in scx_idle_update_selcpu_topology()
367 cpumask_pr_args(llc_span(cpu)), llc_weight(cpu)); in scx_idle_update_selcpu_topology()
384 nr_cpus = numa_weight(cpu); in scx_idle_update_selcpu_topology()
389 cpumask_pr_args(numa_span(cpu)), nr_cpus); in scx_idle_update_selcpu_topology()
465 s32 cpu; in scx_select_cpu_dfl() local
480 cpu = -EBUSY; in scx_select_cpu_dfl()
533 cpu = smp_processor_id(); in scx_select_cpu_dfl()
534 if (is_prev_allowed && cpus_share_cache(cpu, prev_cpu) && in scx_select_cpu_dfl()
536 cpu = prev_cpu; in scx_select_cpu_dfl()
553 waker_node = scx_cpu_node_if_enabled(cpu); in scx_select_cpu_dfl()
555 cpu_rq(cpu)->scx.local_dsq.nr == 0 && in scx_select_cpu_dfl()
557 !cpumask_empty(idle_cpumask(waker_node)->cpu)) { in scx_select_cpu_dfl()
558 if (cpumask_test_cpu(cpu, allowed)) { in scx_select_cpu_dfl()
559 scx_idle_test_and_clear_cpu(cpu); in scx_select_cpu_dfl()
576 cpu = prev_cpu; in scx_select_cpu_dfl()
584 cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
585 if (cpu >= 0) in scx_select_cpu_dfl()
593 cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
594 if (cpu >= 0) in scx_select_cpu_dfl()
606 cpu = scx_pick_idle_cpu(allowed, node, flags | SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
607 if (cpu >= 0) in scx_select_cpu_dfl()
615 cpu = -EBUSY; in scx_select_cpu_dfl()
624 cpu = prev_cpu; in scx_select_cpu_dfl()
632 for_each_cpu_and(cpu, cpu_smt_mask(prev_cpu), allowed) { in scx_select_cpu_dfl()
633 if (cpu == prev_cpu) in scx_select_cpu_dfl()
635 if (scx_idle_test_and_clear_cpu(cpu)) in scx_select_cpu_dfl()
644 cpu = pick_idle_cpu_in_node(llc_cpus, node, 0); in scx_select_cpu_dfl()
645 if (cpu >= 0) in scx_select_cpu_dfl()
653 cpu = pick_idle_cpu_in_node(numa_cpus, node, 0); in scx_select_cpu_dfl()
654 if (cpu >= 0) in scx_select_cpu_dfl()
666 cpu = scx_pick_idle_cpu(allowed, node, flags); in scx_select_cpu_dfl()
673 return cpu; in scx_select_cpu_dfl()
684 BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.cpu, GFP_KERNEL)); in scx_idle_init_masks()
696 BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->cpu, GFP_KERNEL, i)); in scx_idle_init_masks()
711 static void update_builtin_idle(int cpu, bool idle) in update_builtin_idle() argument
713 int node = scx_cpu_node_if_enabled(cpu); in update_builtin_idle()
714 struct cpumask *idle_cpus = idle_cpumask(node)->cpu; in update_builtin_idle()
716 assign_cpu(cpu, idle_cpus, idle); in update_builtin_idle()
719 const struct cpumask *smt = cpu_smt_mask(cpu); in update_builtin_idle()
745 s32 cpu = cpu_of(rq); in scx_idle_notify() local
746 s32 cid = scx_cpu_arg(cpu); in scx_idle_notify()
755 SCX_HAS_OP(root, update_idle) && !scx_bypassing(root, cpu)) in scx_idle_notify()
764 if (unlikely(scx_missing_caps(pos, cpu, SCX_CAP_BASE))) { in scx_idle_notify()
773 else if (per_cpu_ptr(pos->pcpu, cpu)->idle_renotify) { in scx_idle_notify()
774 per_cpu_ptr(pos->pcpu, cpu)->idle_renotify = false; in scx_idle_notify()
779 !scx_bypassing(pos, cpu)) in scx_idle_notify()
803 int cpu = cpu_of(rq); in __scx_update_idle() local
813 update_builtin_idle(cpu, idle); in __scx_update_idle()
849 cpumask_clear(idle_cpumask(NUMA_NO_NODE)->cpu); in reset_idle_masks()
855 cpumask_clear(idle_cpumask(node)->cpu); in reset_idle_masks()
961 s32 cpu; in select_cpu_from_kfunc() local
1004 cpu = prev_cpu; in select_cpu_from_kfunc()
1006 cpu = -EBUSY; in select_cpu_from_kfunc()
1008 cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, in select_cpu_from_kfunc()
1015 return cpu; in select_cpu_from_kfunc()
1029 __bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux) in scx_bpf_cpu_node() argument
1036 if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_cpu_node()
1038 return cpu_to_node(cpu); in scx_bpf_cpu_node()
1062 s32 cpu; in scx_bpf_select_cpu_dfl() local
1070 cpu = select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, NULL, 0); in scx_bpf_select_cpu_dfl()
1071 if (cpu >= 0) { in scx_bpf_select_cpu_dfl()
1073 return cpu; in scx_bpf_select_cpu_dfl()
1182 return idle_cpumask(node)->cpu; in scx_bpf_get_idle_cpumask_node()
1211 return idle_cpumask(NUMA_NO_NODE)->cpu; in scx_bpf_get_idle_cpumask()
1243 return idle_cpumask(node)->cpu; in scx_bpf_get_idle_smtmask_node()
1276 return idle_cpumask(NUMA_NO_NODE)->cpu; in scx_bpf_get_idle_smtmask()
1305 __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_aux *aux) in scx_bpf_test_and_clear_cpu_idle() argument
1318 if (!scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_test_and_clear_cpu_idle()
1321 return scx_idle_test_and_clear_cpu(cpu); in scx_bpf_test_and_clear_cpu_idle()
1435 s32 cpu; in scx_bpf_pick_any_cpu_node() local
1447 cpu = scx_pick_idle_cpu(cpus_allowed, node, flags); in scx_bpf_pick_any_cpu_node()
1448 if (cpu >= 0) in scx_bpf_pick_any_cpu_node()
1449 return cpu; in scx_bpf_pick_any_cpu_node()
1452 cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed); in scx_bpf_pick_any_cpu_node()
1454 cpu = cpumask_any_distribute(cpus_allowed); in scx_bpf_pick_any_cpu_node()
1455 if (cpu < nr_cpu_ids) in scx_bpf_pick_any_cpu_node()
1456 return cpu; in scx_bpf_pick_any_cpu_node()
1483 s32 cpu; in scx_bpf_pick_any_cpu() local
1497 cpu = scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); in scx_bpf_pick_any_cpu()
1498 if (cpu >= 0) in scx_bpf_pick_any_cpu()
1499 return cpu; in scx_bpf_pick_any_cpu()
1502 cpu = cpumask_any_distribute(cpus_allowed); in scx_bpf_pick_any_cpu()
1503 if (cpu < nr_cpu_ids) in scx_bpf_pick_any_cpu()
1504 return cpu; in scx_bpf_pick_any_cpu()