Lines Matching full:cpu
5 * Built-in idle CPU tracking policy.
17 /* Enable/disable built-in idle CPU selection policy */
36 cpumask_var_t cpu; member
52 * Local per-CPU cpumasks (used to generate temporary idle cpumasks).
69 * Returns the NUMA node ID associated with a @cpu, or NUMA_NO_NODE if
72 static int scx_cpu_node_if_enabled(int cpu) in scx_cpu_node_if_enabled() argument
77 return cpu_to_node(cpu); in scx_cpu_node_if_enabled()
80 static bool scx_idle_test_and_clear_cpu(int cpu) in scx_idle_test_and_clear_cpu() argument
82 int node = scx_cpu_node_if_enabled(cpu); in scx_idle_test_and_clear_cpu()
83 struct cpumask *idle_cpus = idle_cpumask(node)->cpu; in scx_idle_test_and_clear_cpu()
86 * SMT mask should be cleared whether we can claim @cpu or not. The SMT in scx_idle_test_and_clear_cpu()
91 const struct cpumask *smt = cpu_smt_mask(cpu); in scx_idle_test_and_clear_cpu()
95 * If offline, @cpu is not its own sibling and in scx_idle_test_and_clear_cpu()
97 * @cpu is never cleared from the idle SMT mask. Ensure that in scx_idle_test_and_clear_cpu()
98 * @cpu is eventually cleared. in scx_idle_test_and_clear_cpu()
106 else if (cpumask_test_cpu(cpu, idle_smts)) in scx_idle_test_and_clear_cpu()
107 __cpumask_clear_cpu(cpu, idle_smts); in scx_idle_test_and_clear_cpu()
110 return cpumask_test_and_clear_cpu(cpu, idle_cpus); in scx_idle_test_and_clear_cpu()
114 * Pick an idle CPU in a specific NUMA node.
118 int cpu; in pick_idle_cpu_in_node() local
122 cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed); in pick_idle_cpu_in_node()
123 if (cpu < nr_cpu_ids) in pick_idle_cpu_in_node()
130 cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed); in pick_idle_cpu_in_node()
131 if (cpu >= nr_cpu_ids) in pick_idle_cpu_in_node()
135 if (scx_idle_test_and_clear_cpu(cpu)) in pick_idle_cpu_in_node()
136 return cpu; in pick_idle_cpu_in_node()
144 * CPU across all available nodes.
149 * Search for an idle CPU across all nodes, excluding @node.
154 s32 cpu = -EBUSY; in pick_idle_cpu_from_online_nodes() local
173 * SCX_OPS_BUILTIN_IDLE_PER_NODE and it's requesting an idle CPU in pick_idle_cpu_from_online_nodes()
182 cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); in pick_idle_cpu_from_online_nodes()
183 if (cpu >= 0) in pick_idle_cpu_from_online_nodes()
188 return cpu; in pick_idle_cpu_from_online_nodes()
199 * Find an idle CPU in the system, starting from @node.
203 s32 cpu; in scx_pick_idle_cpu() local
210 cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); in scx_pick_idle_cpu()
211 if (cpu >= 0) in scx_pick_idle_cpu()
212 return cpu; in scx_pick_idle_cpu()
229 * Return the amount of CPUs in the same LLC domain of @cpu (or zero if the LLC
232 static unsigned int llc_weight(s32 cpu) in llc_weight() argument
236 sd = rcu_dereference(per_cpu(sd_llc, cpu)); in llc_weight()
244 * Return the cpumask representing the LLC domain of @cpu (or NULL if the LLC
247 static struct cpumask *llc_span(s32 cpu) in llc_span() argument
251 sd = rcu_dereference(per_cpu(sd_llc, cpu)); in llc_span()
259 * Return the amount of CPUs in the same NUMA domain of @cpu (or zero if the
262 static unsigned int numa_weight(s32 cpu) in numa_weight() argument
267 sd = rcu_dereference(per_cpu(sd_numa, cpu)); in numa_weight()
278 * Return the cpumask representing the NUMA domain of @cpu (or NULL if the NUMA
281 static struct cpumask *numa_span(s32 cpu) in numa_span() argument
286 sd = rcu_dereference(per_cpu(sd_numa, cpu)); in numa_span()
302 int cpu; in llc_numa_mismatch() local
309 * topologies, CPU hotplugging or virtualized environments can result in llc_numa_mismatch()
322 * In this case, if we only check the first online CPU (cpu0), we might in llc_numa_mismatch()
327 for_each_online_cpu(cpu) in llc_numa_mismatch()
328 if (llc_weight(cpu) != numa_weight(cpu)) in llc_numa_mismatch()
338 * cache-aware / NUMA-aware scheduling optimizations in the default CPU idle
342 * CPU belongs to a single LLC domain, and that each LLC domain is entirely
349 s32 cpu = cpumask_first(cpu_online_mask); in scx_idle_update_selcpu_topology() local
354 * single LLC domain, the idle CPU selection logic can choose any in scx_idle_update_selcpu_topology()
355 * online CPU without bias. in scx_idle_update_selcpu_topology()
358 * online CPU to determine whether a single LLC domain includes all in scx_idle_update_selcpu_topology()
362 nr_cpus = llc_weight(cpu); in scx_idle_update_selcpu_topology()
367 cpumask_pr_args(llc_span(cpu)), llc_weight(cpu)); in scx_idle_update_selcpu_topology()
377 * for an idle CPU in the same domain twice is redundant. in scx_idle_update_selcpu_topology()
384 nr_cpus = numa_weight(cpu); in scx_idle_update_selcpu_topology()
389 cpumask_pr_args(numa_span(cpu)), nr_cpus); in scx_idle_update_selcpu_topology()
418 * Built-in CPU idle selection policy:
424 * 2. Reuse the same CPU:
425 * - prefer the last used CPU to take advantage of cached data (L1, L2) and
434 * 4. Pick a CPU within the same LLC (Last-Level Cache):
435 * - if the above conditions aren't met, pick a CPU that shares the same
439 * 5. Pick a CPU within the same NUMA node, if enabled:
440 * - choose a CPU from the same NUMA node, if the node cpumask is a
443 * 6. Pick any idle CPU within the @cpus_allowed domain.
453 * Return the picked CPU if idle, or a negative value otherwise.
455 * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because
465 s32 cpu; in scx_select_cpu_dfl() local
480 cpu = -EBUSY; in scx_select_cpu_dfl()
487 * we can still try selecting a nearby CPU. in scx_select_cpu_dfl()
524 * If WAKE_SYNC, try to migrate the wakee to the waker's CPU. in scx_select_cpu_dfl()
530 * If the waker's CPU is cache affine and prev_cpu is idle, in scx_select_cpu_dfl()
533 cpu = smp_processor_id(); in scx_select_cpu_dfl()
534 if (is_prev_allowed && cpus_share_cache(cpu, prev_cpu) && in scx_select_cpu_dfl()
536 cpu = prev_cpu; in scx_select_cpu_dfl()
553 waker_node = scx_cpu_node_if_enabled(cpu); in scx_select_cpu_dfl()
555 cpu_rq(cpu)->scx.local_dsq.nr == 0 && in scx_select_cpu_dfl()
557 !cpumask_empty(idle_cpumask(waker_node)->cpu)) { in scx_select_cpu_dfl()
558 if (cpumask_test_cpu(cpu, allowed)) { in scx_select_cpu_dfl()
559 scx_idle_test_and_clear_cpu(cpu); in scx_select_cpu_dfl()
566 * If CPU has SMT, any wholly idle CPU is likely a better pick than in scx_select_cpu_dfl()
576 cpu = prev_cpu; in scx_select_cpu_dfl()
584 cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
585 if (cpu >= 0) in scx_select_cpu_dfl()
593 cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
594 if (cpu >= 0) in scx_select_cpu_dfl()
601 * If the node-aware idle CPU selection policy is enabled in scx_select_cpu_dfl()
606 cpu = scx_pick_idle_cpu(allowed, node, flags | SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
607 if (cpu >= 0) in scx_select_cpu_dfl()
615 cpu = -EBUSY; in scx_select_cpu_dfl()
624 cpu = prev_cpu; in scx_select_cpu_dfl()
632 for_each_cpu_and(cpu, cpu_smt_mask(prev_cpu), allowed) { in scx_select_cpu_dfl()
633 if (cpu == prev_cpu) in scx_select_cpu_dfl()
635 if (scx_idle_test_and_clear_cpu(cpu)) in scx_select_cpu_dfl()
641 * Search for any idle CPU in the same LLC domain. in scx_select_cpu_dfl()
644 cpu = pick_idle_cpu_in_node(llc_cpus, node, 0); in scx_select_cpu_dfl()
645 if (cpu >= 0) in scx_select_cpu_dfl()
650 * Search for any idle CPU in the same NUMA node. in scx_select_cpu_dfl()
653 cpu = pick_idle_cpu_in_node(numa_cpus, node, 0); in scx_select_cpu_dfl()
654 if (cpu >= 0) in scx_select_cpu_dfl()
659 * Search for any idle CPU usable by the task. in scx_select_cpu_dfl()
661 * If the node-aware idle CPU selection policy is enabled in scx_select_cpu_dfl()
666 cpu = scx_pick_idle_cpu(allowed, node, flags); in scx_select_cpu_dfl()
673 return cpu; in scx_select_cpu_dfl()
684 BUG_ON(!alloc_cpumask_var(&scx_idle_global_masks.cpu, GFP_KERNEL)); in scx_idle_init_masks()
696 BUG_ON(!alloc_cpumask_var_node(&scx_idle_node_masks[i]->cpu, GFP_KERNEL, i)); in scx_idle_init_masks()
700 /* Allocate local per-cpu idle cpumasks */ in scx_idle_init_masks()
711 static void update_builtin_idle(int cpu, bool idle) in update_builtin_idle() argument
713 int node = scx_cpu_node_if_enabled(cpu); in update_builtin_idle()
714 struct cpumask *idle_cpus = idle_cpumask(node)->cpu; in update_builtin_idle()
716 assign_cpu(cpu, idle_cpus, idle); in update_builtin_idle()
719 const struct cpumask *smt = cpu_smt_mask(cpu); in update_builtin_idle()
737 * Notify schedulers of an idle transition on @cpu's cid, delivering to every
745 s32 cpu = cpu_of(rq); in scx_idle_notify() local
746 s32 cid = scx_cpu_arg(cpu); in scx_idle_notify()
755 SCX_HAS_OP(root, update_idle) && !scx_bypassing(root, cpu)) in scx_idle_notify()
764 if (unlikely(scx_missing_caps(pos, cpu, SCX_CAP_BASE))) { in scx_idle_notify()
773 else if (per_cpu_ptr(pos->pcpu, cpu)->idle_renotify) { in scx_idle_notify()
774 per_cpu_ptr(pos->pcpu, cpu)->idle_renotify = false; in scx_idle_notify()
779 !scx_bypassing(pos, cpu)) in scx_idle_notify()
786 * Update the idle state of a CPU to @idle.
793 * This distinction is necessary, because an idle CPU can be "reserved" and
795 * busy even if no tasks are dispatched. In this case, the CPU may return
803 int cpu = cpu_of(rq); in __scx_update_idle() local
813 update_builtin_idle(cpu, idle); in __scx_update_idle()
844 * bypass is lifted and each idle CPU is forced through an idle re-pick. in reset_idle_masks()
845 * This may temporarily omit idle CPUs but never advertises a busy CPU as in reset_idle_masks()
849 cpumask_clear(idle_cpumask(NUMA_NO_NODE)->cpu); in reset_idle_masks()
855 cpumask_clear(idle_cpumask(node)->cpu); in reset_idle_masks()
961 s32 cpu; in select_cpu_from_kfunc() local
997 * per-CPU tasks as well. For these tasks, we can skip all idle CPU in select_cpu_from_kfunc()
999 * used CPU is idle and within the allowed cpumask. in select_cpu_from_kfunc()
1004 cpu = prev_cpu; in select_cpu_from_kfunc()
1006 cpu = -EBUSY; in select_cpu_from_kfunc()
1008 cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, in select_cpu_from_kfunc()
1015 return cpu; in select_cpu_from_kfunc()
1024 * scx_bpf_cpu_node - Return the NUMA node the given @cpu belongs to, or
1025 * trigger an error if @cpu is invalid
1026 * @cpu: target CPU
1029 __bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux) in scx_bpf_cpu_node() argument
1036 if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_cpu_node()
1038 return cpu_to_node(cpu); in scx_bpf_cpu_node()
1043 * @p: task_struct to select a CPU for
1044 * @prev_cpu: CPU @p was on previously
1046 * @is_idle: out parameter indicating whether the returned CPU is idle
1050 * context such as a BPF test_run() call, as long as built-in CPU selection
1054 * Returns the picked CPU with *@is_idle indicating whether the picked CPU is
1062 s32 cpu; in scx_bpf_select_cpu_dfl() local
1070 cpu = select_cpu_from_kfunc(sch, p, prev_cpu, wake_flags, NULL, 0); in scx_bpf_select_cpu_dfl()
1071 if (cpu >= 0) { in scx_bpf_select_cpu_dfl()
1073 return cpu; in scx_bpf_select_cpu_dfl()
1087 * __scx_bpf_select_cpu_and - Arg-wrapped CPU selection with cpumask
1088 * @p: task_struct to select a CPU for
1091 * @args->prev_cpu: CPU @p was on previously
1101 * context such as a BPF test_run() call, as long as built-in CPU selection
1107 * Returns the selected idle CPU, which will be automatically awakened upon
1109 * a negative value if no idle CPU is available.
1166 * idle-tracking per-CPU cpumask of a target NUMA node.
1189 return idle_cpumask(node)->cpu; in scx_bpf_get_idle_cpumask_node()
1194 * per-CPU cpumask.
1218 return idle_cpumask(NUMA_NO_NODE)->cpu; in scx_bpf_get_idle_cpumask()
1250 return idle_cpumask(node)->cpu; in scx_bpf_get_idle_smtmask_node()
1283 return idle_cpumask(NUMA_NO_NODE)->cpu; in scx_bpf_get_idle_smtmask()
1302 * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state
1303 * @cpu: cpu to test and clear idle for
1306 * Returns %true if @cpu was idle and its idle state was successfully cleared.
1312 __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_aux *aux) in scx_bpf_test_and_clear_cpu_idle() argument
1325 if (!scx_cpu_valid(sch, cpu, NULL)) in scx_bpf_test_and_clear_cpu_idle()
1328 return scx_idle_test_and_clear_cpu(cpu); in scx_bpf_test_and_clear_cpu_idle()
1332 * scx_bpf_pick_idle_cpu_node - Pick and claim an idle cpu from @node
1338 * Pick and claim an idle cpu in @cpus_allowed from the NUMA node @node.
1340 * Returns the picked idle cpu number on success, or -%EBUSY if no matching
1341 * cpu was found.
1371 * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu
1376 * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu
1377 * number on success. -%EBUSY if no matching cpu was found.
1379 * Idle CPU tracking may race against CPU scheduling state transitions. For
1416 * scx_bpf_pick_any_cpu_node - Pick and claim an idle cpu if available
1417 * or pick any CPU from @node
1423 * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any
1424 * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu
1431 * the CPU idle state).
1435 * CPU.
1442 s32 cpu; in scx_bpf_pick_any_cpu_node() local
1454 cpu = scx_pick_idle_cpu(cpus_allowed, node, flags); in scx_bpf_pick_any_cpu_node()
1455 if (cpu >= 0) in scx_bpf_pick_any_cpu_node()
1456 return cpu; in scx_bpf_pick_any_cpu_node()
1459 cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed); in scx_bpf_pick_any_cpu_node()
1461 cpu = cpumask_any_distribute(cpus_allowed); in scx_bpf_pick_any_cpu_node()
1462 if (cpu < nr_cpu_ids) in scx_bpf_pick_any_cpu_node()
1463 return cpu; in scx_bpf_pick_any_cpu_node()
1469 * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU
1474 * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any
1475 * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu
1481 * CPU.
1490 s32 cpu; in scx_bpf_pick_any_cpu() local
1504 cpu = scx_pick_idle_cpu(cpus_allowed, NUMA_NO_NODE, flags); in scx_bpf_pick_any_cpu()
1505 if (cpu >= 0) in scx_bpf_pick_any_cpu()
1506 return cpu; in scx_bpf_pick_any_cpu()
1509 cpu = cpumask_any_distribute(cpus_allowed); in scx_bpf_pick_any_cpu()
1510 if (cpu < nr_cpu_ids) in scx_bpf_pick_any_cpu()
1511 return cpu; in scx_bpf_pick_any_cpu()