Lines Matching +full:locality +full:- +full:specific
1 /* SPDX-License-Identifier: GPL-2.0 */
3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
17 * scx_add_event - Increase an event counter for 'name' by 'cnt'
25 this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \
30 * __scx_add_event - Increase an event counter for 'name' by 'cnt'
38 __this_cpu_add((sch)->pcpu->event_stats.name, (cnt)); \
49 SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */
50 SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */
51 SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */
71 * SYS ACT: System-defined exit actions
72 * SYS RSN: System-defined exit reasons
73 * USR : User-defined exit codes and reasons
76 * actions and/or system reasons with a user-defined exit code.
102 /* %SCX_EXIT_* - broad category of the exit reason */
134 * Keep built-in idle tracking even if ops.update_idle() is implemented.
173 * only select the current CPU. Also, p->cpus_ptr will only contain its
174 * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr
175 * and thus may disagree with cpumask_weight(p->cpus_ptr).
182 * previous CPU via IPI (inter-processor interrupt) to reduce cacheline
196 * If set, enable per-node idle cpumasks. If clear, use a single global
208 * Maintain a mapping from p->scx.tid to task_struct so the BPF
212 * Only the root scheduler turns this on. A sub-sched may set the flag
214 * enabled it, attaching the sub-sched is rejected.
315 * struct sched_ext_ops - Operation table for BPF scheduler implementation
376 * on the scheduling logic, this can lead to confusing behaviors - e.g.
398 * @prev->scx.flags, it is not enqueued yet and will be enqueued after
421 * execution state transitions. A task becomes ->runnable() on a CPU,
422 * and then goes through one or more ->running() and ->stopping() pairs
423 * as it runs on the CPU, and eventually becomes ->quiescent() when it's
428 * - waking up (%SCX_ENQ_WAKEUP)
429 * - being moved from another CPU
430 * - being restored after temporarily taken off the queue for an
433 * This and ->enqueue() are related but not coupled. This operation
434 * notifies @p's state transition and may not be followed by ->enqueue()
437 * task may be ->enqueue()'d without being preceded by this operation
455 * See ->runnable() for explanation on the task state notifiers.
473 * See ->runnable() for explanation on the task state notifiers. If
474 * !@runnable, ->quiescent() will be invoked after this operation
484 * See ->runnable() for explanation on the task state notifiers.
488 * - sleeping (%SCX_DEQ_SLEEP)
489 * - being moved to another CPU
490 * - being temporarily taken off the queue for an attribute change
493 * This and ->dequeue() are related but not coupled. This operation
494 * notifies @p's state transition and may not be preceded by ->dequeue()
509 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf
515 * @core_sched_before: Task ordering for core-sched
519 * Used by core-sched to determine the ordering between two tasks. See
520 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on
521 * core-sched.
529 * called on tasks that the scheduler delegated to its sub-schedulers
562 * state. By default, implementing this operation disables the built-in
565 * - scx_bpf_select_cpu_dfl()
566 * - scx_bpf_select_cpu_and()
567 * - scx_bpf_test_and_clear_cpu_idle()
568 * - scx_bpf_pick_idle_cpu()
573 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
593 * Return 0 for success, -errno for failure. An error return while
595 * will abort that specific fork.
600 * @exit_task: Exit a previously-running task from the system
632 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump.
642 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for
653 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for
668 * Cgroup handovers also generate these ops: an enabling sub-scheduler
678 * Return 0 for success, -errno for failure. An error return while
680 * creation, it will abort the specific cgroup creation.
709 * that don't re-home @p. A re-homing move is reported through
712 * the cpu controller can be coarser than the sub-scheduler topology.
714 * Return 0 for success, -errno for failure. An error return aborts the
752 * ops.cgroup_init() for @cgrp - at a sub-scheduler attach point, the
771 * burst temporarily. The specific control mechanism and thus the
786 * a cgroup transitions between idle and non-idle states, allowing the
796 * @sub_attach: Attach a sub-scheduler
799 * Return 0 to accept the sub-scheduler. -errno to reject.
804 * @sub_detach: Detach a sub-scheduler
810 * @sub_caps_updated: Caps on this sub-sched's shard changed
811 * @cmask: cids whose caps changed (cmask->base identifies the shard)
815 * only one in-flight invocation per shard. @cmask and @caps coalesce
821 * the sub-sched holds and propagate to its own children, not to decide
830 * @sub_ecaps_updated: This sub-sched's effective caps on a cid changed
835 * Invoked when this sub-sched's effective caps on @cid change, once the
869 * @init_cids: Finalize the cid layout (cid-form only)
872 * are finalized. A cid-form scheduler may call scx_bpf_cid_override()
873 * here for a custom layout. Ignored for cpu-form schedulers.
934 * domains for sub-scheduling. Each LLC is divided into ceil(nr_cpus /
950 * (2%). May not exceed 250 (25%). %SCX_RESCUE_DISABLE disables rescue -
961 * @rescue_bandwidth_ppt - with the defaults, one 5ms rescue every
962 * 250ms. A crowded queue round-robins on the quantum divided across the
968 * Non-zero values must be within [1000, 100000]. Values too short for
974 * @sub_cgroup_id: When >1, attach the scheduler as a sub-scheduler
982 * Must be a non-zero valid BPF object name including only isalnum(),
993 * Deprecated callbacks. Kept at the end of the struct so the cid-form
997 * be used; callers must test sch->ops.cpu_acquire / cpu_release
1019 * caller should consult @args->reason to determine the cause.
1026 * struct sched_ext_ops_cid - cid-form alternative to struct sched_ext_ops
1030 * byte-for-byte (verified by BUILD_BUG_ON checks at scx_init() time) so
1036 * - select_cpu -> select_cid (returns cid)
1037 * - dispatch -> dispatch (cpu arg is now cid)
1038 * - update_idle -> update_idle (cpu arg is now cid)
1039 * - set_cpumask -> set_cmask (cmask instead of cpumask)
1040 * - cpu_online -> cid_online
1041 * - cpu_offline -> cid_offline
1042 * - dump_cpu -> dump_cid
1043 * - cgroup_* -> cpuctl_* (they track the cgroup cpu controller)
1044 * - cpu_acquire/cpu_release -> not present (deprecated in sched_ext_ops)
1046 * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs;
1047 * use the cid-form variants instead. Enforced at BPF verifier time via
1048 * scx_kfunc_context_filter() branching on prog->aux->st_ops.
1101 /* Data fields - must match sched_ext_ops layout exactly */
1166 * SCX_ENQ_IMMED, was re-enqueued because the CPU was not available for
1174 * rapidly indicates that the BPF scheduler keeps re-deciding placements
1187 * The number of times an out-of-band slice request exceeded the maximum
1219 * ignored dequeue around sub-sched enabling. If this count keeps going
1220 * up regardless of sub-sched enabling, it likely indicates a bug in the
1232 * The number of times a migration-disabled task lacking the cap for its
1240 * sub-sched lacked SCX_CAP_PREEMPT for a task outside its subtree. The
1246 * The number of times a kick was skipped because the sub-sched lacked
1247 * baseline access on the target cid. The preempt-part degradation of a
1254 * sub-sched lacked baseline access on the target cid.
1260 * sub-sched lacked SCX_CAP_PERF on the target cid.
1329 * rq->scx.sched_pcpus_to_kick. The cpu's single kick irq_work walks
1330 * that list and kicks each sched's targets on its behalf. Per-sched so
1341 * pshard->caps[cap_bit] is the set of cids the sched holds that one
1344 * the hot-path check is a single read.
1346 * While pshard->caps[] under pshard->lock is the target configuration,
1348 * rq lock while processing rq->ecaps_to_sync. Can also be read with
1355 /* owed a forced update_idle() re-notify on this cpu */
1370 * The event counters are in a per-CPU variable to minimize the
1371 * accounting overhead. A system-wide view on the event counter is
1382 /* must be the last entry - contains flex array */
1391 * Sub-sched capability delegation.
1393 * Caps are per-cid permissions parents delegate to direct children via
1395 * subset of its parent's. A sub-sched checks its caps locally, and cross-sched
1398 * Caps are used to implement sub-sched scheduling on the enqueue path. Picking
1400 * Resolving that programmatically on every enqueue would mean a cross-sched
1401 * round-trip call chain, possibly retrying if the request can't be granted
1402 * as-is.
1404 * The dispatch path is different - it runs as top-down recursion via
1408 * Locking is per shard. cid space is split into shards, and each sub-sched has
1409 * its own pshard->lock for each shard. Operations are broken up on shard
1411 * topology-aligned and likely to serve as the locality unit when cids are
1412 * allocated to schedulers, so per-shard lock granularity scales naturally with
1416 * - kick the cid's cpu (except SCX_KICK_PREEMPT)
1423 * - SCX_ENQ_PREEMPT inserts
1424 * - SCX_KICK_PREEMPT kicks
1432 * scx_bpf_sub_grant() delegates literally-held caps, so a cap held only through
1433 * implication is usable but cannot be re-delegated to a child. When granting a
1444 __SCX_CAP_ALL = BIT_U64(__SCX_NR_CAPS) - 1,
1463 __caps &= __caps - 1)
1466 * Sub-cap update notifier.
1468 * ops_cid.sub_caps_updated() notifies sub-scheds when their cap state changes
1474 * 1. Static memory. Deliveries use a fixed-size buffer, both for runtime
1477 * 2. High-frequency updates. Grant/revoke can mutate caps in bursts, and the
1486 * Direction (set vs cleared) isn't encoded as it doesn't fit in the fixed-size
1499 /* Kernel-side accumulator. Access as &cu->cmask. */
1511 * Per-cap cmask, inline via TRAILING_OVERLAP so cmask.bits[] overlaps
1513 * scx_sched_pcpu->ecaps.
1521 * bpf-writable and the live shard range can change before the
1522 * rcu-deferred free, so re-init and size cmask_arena_out from these
1532 * cpu-form and cid-form ops share field offsets up to .priv (verified
1534 * access either view of the same storage without function-pointer
1535 * casts: use .ops for cpu-form and shared fields, .ops_cid for the
1536 * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...).
1546 * Arena map auto-discovered from member progs at struct_ops attach.
1547 * cid-form schedulers must use exactly one arena across all member
1548 * progs. NULL on cpu-form.
1550 * @arena_pool sub-allocates @arena_map. Each gen_pool chunk is added
1551 * at the kernel-side mapping address. @arena_kern_base is the start
1559 * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask
1571 * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability.
1572 * This is to avoid live-locking in bypass mode where all tasks are
1574 * per-node split isn't sufficient, it can be further split.
1598 * pshard[] size captured at enable for the async RCU free path -
1627 bool linked; /* on ->children, see scx_link_sched() */
1658 * scx_arena_to_kaddr - Translate a BPF-arena pointer to its kernel address
1660 * @bpf_ptr: BPF-arena pointer, only the low 32 bits are used
1663 * which combined with scratch-page fault recovery makes the returned pointer
1665 * larger than GUARD_SZ / 2 must be explicitly bounds-checked.
1669 return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr);
1685 /* high 32bits are SCX specific */
1701 * The protection persists until the next fresh enqueue - it
1706 * Exiting and migration-disabled tasks bypass ops.enqueue() and
1719 * make forward progress and is bandwidth-limited, see the
1726 * removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find
1737 * The BPF scheduler is responsible for triggering a follow-up
1758 /* high 32bits are SCX specific */
1761 * The generic core-sched layer decided to execute the task even though
1778 /* internal: kernel-issued on cap revoke, not accepted from BPF */
1785 /* bits 32-35 used by task_should_reenq() */
1808 * current task of the target CPU is an SCX task, its ->scx.slice is
1844 * Task Ownership State Machine (sched_ext_entity->ops_state)
1853 * .------------> NONE (owned by SCX core)
1857 * | QUEUEING -------'
1871 * `---------------'
1875 * - %SCX_OPSS_NONE:
1880 * - %SCX_OPSS_QUEUEING:
1886 * path busy-waits for it to leave this state (via wait_ops_state())
1889 * - %SCX_OPSS_QUEUED:
1893 * dispatch/dequeue races: if a task is dequeued and re-enqueued, the
1894 * QSEQ changes and any in-flight dispatch operations targeting the old
1897 * - %SCX_OPSS_DISPATCHING:
1902 * busy-waits for it to leave this state (via wait_ops_state()) before
1912 * Cross-CPU Task Migration
1920 * p->scx.holding_cpu. When moving a task to a different rq:
1922 * 1. Set p->scx.holding_cpu to the current CPU
1928 * 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
1929 * race (dequeue clears holding_cpu to -1 when it takes the task), in
1940 * 9. Unlock dst_rq and re-lock the current CPU's rq to restore
1962 * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on
1970 #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1)
1991 * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid
1992 * starvation. During the READY -> ENABLED task switching loop, the calling
1995 * fair-class saturation, leading to a system hang.
2019 * DEFINE_SCX_COMPAT_MARKER() - define a userspace capability marker
2023 * Its presence is part of the kernel<->userspace contract: userspace probes
2027 * The leading declaration suppresses the missing-prototype warning; the
2039 * True when the currently loaded scheduler hierarchy is cid-form. All scheds
2041 * view to use without per-sch dereferences. Use scx_is_cid_type() to test.
2126 * scx_root_protected_live - Root sched for paths that only run while live
2139 * scx_root_protected - Root sched for contexts that exclude its updates
2154 return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq;
2158 * scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled
2162 * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors
2174 return unlikely(atomic_read(&sch->bypass_dsp_enable_depth));
2178 * scx_ops_sanitize_err - Sanitize a -errno value
2181 * @err: -errno value to sanitize
2183 * Verify @err is a valid -errno. If not, trigger scx_error() and return
2184 * -%EPROTO. This is necessary because returning a rogue -errno up the chain can
2192 if (err < 0 && err >= -MAX_ERRNO)
2196 return -EPROTO;
2206 schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq);
2230 #define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op)
2233 * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
2247 (sch)->ops.op(args); \
2258 __typeof__((sch)->ops.op(args)) __ret; \
2264 __ret = (sch)->ops.op(args); \
2272 * and records them in current->scx.kf_tasks[] for the duration of the call. A
2277 * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
2280 * So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
2282 * kf_tasks[] can not stack, so task-based SCX ops must not nest. The
2283 * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
2288 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
2289 current->scx.kf_tasks[0] = task; \
2291 current->scx.kf_tasks[0] = NULL; \
2295 * A per-task op runs on @task's owner - WARN if @sch isn't it. Sites that must
2305 * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs
2307 * invoked via its cpu-form set_cpumask() slot.
2314 __typeof__((sch)->ops.op(task, ##args)) __ret; \
2316 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
2317 current->scx.kf_tasks[0] = task; \
2319 current->scx.kf_tasks[0] = NULL; \
2325 __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
2326 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
2327 current->scx.kf_tasks[0] = task0; \
2328 current->scx.kf_tasks[1] = task1; \
2330 current->scx.kf_tasks[0] = NULL; \
2331 current->scx.kf_tasks[1] = NULL; \
2339 if (unlikely((p != current->scx.kf_tasks[0] &&
2340 p != current->scx.kf_tasks[1]))) {
2350 return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags &
2358 * scx_has_subs - Whether any sub-scheduler exists
2360 * Gates the sub-sched portions of hot paths so that a root-only system doesn't
2369 * scx_task_sched - Find scx_sched scheduling a task
2377 return rcu_dereference_protected(p->scx.sched,
2378 lockdep_is_held(&p->pi_lock) ||
2383 * scx_task_sched_rcu - Find scx_sched scheduling a task
2390 return rcu_dereference_all(p->scx.sched);
2394 * scx_task_on_sched - Is a task on the specified sched?
2403 return rcu_access_pointer(p->scx.sched) == sch;
2407 * scx_prog_sched - Find scx_sched associated with a BPF prog
2421 sch = rcu_dereference_all(ops->priv);
2422 if (sch && unlikely(READ_ONCE(sch->dead)))
2429 if (unlikely(READ_ONCE(root->dead)))
2432 * COMPAT-v6.19: Schedulers built before sub-sched support was
2433 * introduced may have unassociated non-struct_ops programs.
2435 if (!root->ops.sub_attach)
2438 if (!root->warned_unassoc_progs) {
2440 aux->name, aux->id);
2441 root->warned_unassoc_progs = true;
2449 * scx_parent - Find the parent sched
2456 if (sch->level)
2457 return sch->ancestors[sch->level - 1];
2468 lockdep_is_held(&p->pi_lock) ||
2487 if (root && unlikely(READ_ONCE(root->dead)))