1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Shared definitions between scx_qmap.bpf.c and scx_qmap.c. 4 * 5 * The scheduler keeps all state in a single BPF arena map. struct 6 * qmap_arena is the one object that lives at the base of the arena and is 7 * mmap'd into userspace so the loader can read counters directly. 8 * 9 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 10 * Copyright (c) 2026 Tejun Heo <tj@kernel.org> 11 */ 12 #ifndef __SCX_QMAP_H 13 #define __SCX_QMAP_H 14 15 #ifdef __BPF__ 16 #include <scx/bpf_arena_common.bpf.h> 17 #else 18 #include <linux/types.h> 19 #include <scx/bpf_arena_common.h> 20 #endif 21 22 #define MAX_SUB_SCHEDS 8 23 #define MAX_PARTS (MAX_SUB_SCHEDS + 1) /* participants: children + self */ 24 25 /* 26 * cpu_ctxs[] is sized to a fixed cap so the layout is shared between BPF and 27 * userspace. Keep this in sync with NR_CPUS used by the BPF side. 28 */ 29 #define SCX_QMAP_MAX_CPUS 1024 30 31 /* 32 * An owner id identifies who holds a cid: a child slot in [0, MAX_SUB_SCHEDS), 33 * CID_SELF for this node, CID_NONE for a cid not currently held, or CID_SHARED 34 * for a cid in the round-robin pool (its live holder is rr_slots[rr_pos]). Used 35 * by the partition's cid_owner[]. 36 */ 37 #define CID_SELF (-1) 38 #define CID_NONE (-2) 39 #define CID_SHARED (-3) 40 41 /* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */ 42 enum qmap_cid_override { 43 QMAP_CID_OVR_OFF = 0, /* disabled */ 44 QMAP_CID_OVR_SHUFFLE = 1, /* valid reversed cpu->cid mapping */ 45 QMAP_CID_OVR_BAD_DUP = 2, /* invalid: duplicate cid assignment */ 46 QMAP_CID_OVR_BAD_RANGE = 3, /* invalid: out-of-range cid */ 47 QMAP_CID_OVR_BAD_MONO = 4, /* invalid: non-monotonic shard_start */ 48 }; 49 50 struct cpu_ctx { 51 u64 dsp_idx; /* dispatch index */ 52 u64 dsp_cnt; /* remaining count */ 53 u32 avg_weight; 54 u32 cpuperf_target; 55 }; 56 57 struct qmap_fifo { 58 struct task_ctx __arena *head; 59 struct task_ctx __arena *tail; 60 s32 idx; 61 }; 62 63 /* -J fault-injection modes. Selects inject_mode in struct qmap_arena. */ 64 enum qmap_inject { 65 QMAP_INJ_OFF = 0, 66 QMAP_INJ_WRONG_CID = 1, /* dispatch to a cid we don't hold */ 67 }; 68 69 /* 70 * scx_cmask's are embedded in struct qmap_arena with inline backing storage. 71 * The bpf side uses &field.mask with the normal cmask_* helpers. Userspace 72 * doesn't have access to the type definition and sees same-sized opaque words. 73 * _Static_assert()'s in .bpf.c ensure that they are in sync. 74 */ 75 #define QMAP_CMASK_WORDS (((SCX_QMAP_MAX_CPUS) + 63) / 64 + 1) 76 struct qmap_cmask { 77 #ifdef __BPF__ 78 union { 79 struct scx_cmask mask; 80 u64 words[QMAP_CMASK_WORDS + 2]; 81 }; 82 #else 83 u64 words[QMAP_CMASK_WORDS + 2]; 84 #endif 85 }; 86 87 /* Opaque to userspace; defined in scx_qmap.bpf.c. */ 88 struct task_ctx; 89 90 /* per-direct-child state for the sub-scheduler */ 91 struct sub_sched_ctx { 92 u64 cgroup_id; 93 u32 weight; /* cpu.weight, fed by userspace */ 94 u64 nr_dsps; 95 struct qmap_cmask granted_cids; /* cids granted excl to this child */ 96 struct qmap_cmask prev_granted; /* last grant, for delta calculation */ 97 }; 98 99 /* 100 * compute_partition() builds the following from this node's held caps, and 101 * apply_partition()/rr_advance() execute it. Userspace only reads for the 102 * hierarchy display. 103 */ 104 struct qmap_partition { 105 u32 nr_excl; /* number of excl-held (delegatable) cids */ 106 s32 cid_owner[SCX_QMAP_MAX_CPUS]; /* per cid: owner id, or CID_NONE */ 107 s32 shared_cids[MAX_PARTS]; /* the round-robin cid pool */ 108 u32 nr_shared; /* number of shared_cids entries */ 109 u64 rr_slots[MAX_PARTS]; /* rotation order: holder cgroup_id, 0 = self */ 110 u32 nr_rr; /* number of rr_slots entries */ 111 u32 rr_pos; /* current rotation index */ 112 }; 113 114 struct qmap_arena { 115 /* userspace-visible stats */ 116 u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0; 117 u64 nr_dequeued, nr_ddsp_from_enq; 118 u64 nr_core_sched_execed; 119 u64 nr_expedited_local, nr_expedited_remote; 120 u64 nr_expedited_lost, nr_expedited_from_timer; 121 u64 nr_highpri_queued; 122 u32 test_error_cnt; 123 u32 cpuperf_min, cpuperf_avg, cpuperf_max; 124 u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max; 125 126 /* kernel-side runtime state */ 127 u64 core_sched_head_seqs[5]; 128 u64 core_sched_tail_seqs[5]; 129 130 struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS]; 131 132 /* task_ctx slab; allocated and threaded by qmap_init() */ 133 struct task_ctx __arena *task_ctxs; 134 struct task_ctx __arena *task_free_head; 135 136 /* five priority FIFOs, each a doubly-linked list through task_ctx */ 137 struct qmap_fifo fifos[5]; 138 139 /* 140 * Hierarchical sub-scheduling state. See the design comment at the top 141 * of scx_qmap.bpf.c. 142 */ 143 u32 nr_cids; /* cid count, cached at init */ 144 145 /* bpf-owned partition: read by userspace for display */ 146 struct qmap_partition part; 147 148 struct sub_sched_ctx sub_sched_ctxs[MAX_SUB_SCHEDS]; /* per-child context */ 149 u64 nr_sub_scheds; /* number of attached children */ 150 u32 self_weight; /* this node's cpu.weight, fed by userspace */ 151 152 /* bpf-internal per-cid state */ 153 u8 cid_shared[SCX_QMAP_MAX_CPUS]; /* per cid: 1 if held shared (ENQ_IMMED-only) */ 154 155 /* allocated cid-time, charged per owner by account_alloc() */ 156 u64 alloc_ns[MAX_SUB_SCHEDS]; /* per child slot */ 157 u64 self_alloc_ns; 158 u64 alloc_ts; /* last accounting timestamp */ 159 u64 alloc_window_ns; /* total accounted time, the alloc denominator */ 160 161 /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ 162 struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ 163 struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */ 164 struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */ 165 166 /* scratch cmasks */ 167 struct qmap_cmask to_revoke_cids; /* delta cids to revoke */ 168 struct qmap_cmask to_grant_cids; /* delta cids to grant */ 169 struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */ 170 struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */ 171 struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */ 172 173 /* bpf -> userspace: stats */ 174 u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */ 175 u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */ 176 u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */ 177 u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */ 178 }; 179 180 #endif /* __SCX_QMAP_H */ 181