1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Shared definitions between scx_qmap.bpf.c and scx_qmap.c. 4 * 5 * The scheduler keeps all state in a single BPF arena map. struct 6 * qmap_arena is the one object that lives at the base of the arena and is 7 * mmap'd into userspace so the loader can read counters directly. 8 * 9 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 10 * Copyright (c) 2026 Tejun Heo <tj@kernel.org> 11 */ 12 #ifndef __SCX_QMAP_H 13 #define __SCX_QMAP_H 14 15 #ifdef __BPF__ 16 #include <scx/bpf_arena_common.bpf.h> 17 #else 18 #include <linux/types.h> 19 #include <scx/bpf_arena_common.h> 20 #endif 21 22 #define MAX_SUB_SCHEDS 8 23 #define MAX_PARTS (MAX_SUB_SCHEDS + 1) /* participants: children + self */ 24 25 /* 26 * cpu_ctxs[] is sized to a fixed cap so the layout is shared between BPF and 27 * userspace. Keep this in sync with NR_CPUS used by the BPF side. 28 */ 29 #define SCX_QMAP_MAX_CPUS 1024 30 31 /* 32 * An owner id identifies who holds a cid: a child slot in [0, MAX_SUB_SCHEDS), 33 * CID_SELF for this node, CID_NONE for a cid not currently held, or CID_SHARED 34 * for a cid in the round-robin pool (its live holder is rr_slots[rr_pos]). Used 35 * by the partition's cid_owner[]. 36 */ 37 #define CID_SELF (-1) 38 #define CID_NONE (-2) 39 #define CID_SHARED (-3) 40 41 /* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */ 42 enum qmap_cid_override { 43 QMAP_CID_OVR_OFF = 0, /* disabled */ 44 QMAP_CID_OVR_SHUFFLE = 1, /* valid reversed cpu->cid mapping */ 45 QMAP_CID_OVR_BAD_DUP = 2, /* invalid: duplicate cid assignment */ 46 QMAP_CID_OVR_BAD_RANGE = 3, /* invalid: out-of-range cid */ 47 QMAP_CID_OVR_BAD_MONO = 4, /* invalid: non-monotonic shard_start */ 48 }; 49 50 struct cpu_ctx { 51 u64 dsp_idx; /* dispatch index */ 52 u64 dsp_cnt; /* remaining count */ 53 u32 avg_weight; 54 u32 cpuperf_target; 55 }; 56 57 struct qmap_fifo { 58 struct task_ctx __arena *head; 59 struct task_ctx __arena *tail; 60 s32 idx; 61 }; 62 63 /* -J fault-injection modes. Selects inject_mode in struct qmap_arena. */ 64 enum qmap_inject { 65 QMAP_INJ_OFF = 0, 66 QMAP_INJ_WRONG_CID = 1, /* dispatch to a cid we don't hold */ 67 QMAP_INJ_INIT_FAIL = 2, /* fail init_task for "qmfail*" comms */ 68 QMAP_INJ_CGRP_INIT_FAIL = 3, /* fail cpuctl_init for "qmfail*" cgroups */ 69 }; 70 71 /* 72 * scx_cmask's are embedded in struct qmap_arena with inline backing storage. 73 * The bpf side uses &field.mask with the normal cmask_* helpers. Userspace 74 * doesn't have access to the type definition and sees same-sized opaque words. 75 * _Static_assert()'s in .bpf.c ensure that they are in sync. 76 */ 77 #define QMAP_CMASK_WORDS (((SCX_QMAP_MAX_CPUS) + 63) / 64 + 1) 78 struct qmap_cmask { 79 #ifdef __BPF__ 80 union { 81 struct scx_cmask mask; 82 u64 words[QMAP_CMASK_WORDS + 2]; 83 }; 84 #else 85 u64 words[QMAP_CMASK_WORDS + 2]; 86 #endif 87 }; 88 89 /* Opaque to userspace; defined in scx_qmap.bpf.c. */ 90 struct task_ctx; 91 92 /* per-direct-child state for the sub-scheduler */ 93 struct sub_sched_ctx { 94 u64 cgroup_id; 95 u32 weight; /* cpu.weight, seeded at attach, then set_weight */ 96 u64 nr_dsps; 97 struct qmap_cmask granted_cids; /* cids granted excl to this child */ 98 struct qmap_cmask prev_granted; /* last grant, for delta calculation */ 99 }; 100 101 /* 102 * compute_partition() builds the following from this node's held caps, and 103 * apply_partition()/rr_advance() execute it. Userspace only reads for the 104 * hierarchy display. 105 */ 106 struct qmap_partition { 107 u32 nr_excl; /* number of excl-held (delegatable) cids */ 108 s32 cid_owner[SCX_QMAP_MAX_CPUS]; /* per cid: owner id, or CID_NONE */ 109 s32 shared_cids[MAX_PARTS]; /* the round-robin cid pool */ 110 u32 nr_shared; /* number of shared_cids entries */ 111 u64 rr_slots[MAX_PARTS]; /* rotation order: holder cgroup_id, 0 = self */ 112 u32 nr_rr; /* number of rr_slots entries */ 113 u32 rr_pos; /* current rotation index */ 114 }; 115 116 struct qmap_arena { 117 /* userspace-visible stats */ 118 u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0; 119 u64 nr_dequeued, nr_ddsp_from_enq; 120 u64 nr_core_sched_execed; 121 u64 nr_expedited_local, nr_expedited_remote; 122 u64 nr_expedited_lost, nr_expedited_from_timer; 123 u64 nr_highpri_queued; 124 u32 test_error_cnt; 125 u32 cpuperf_min, cpuperf_avg, cpuperf_max; 126 u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max; 127 128 /* kernel-side runtime state */ 129 u64 core_sched_head_seqs[5]; 130 u64 core_sched_tail_seqs[5]; 131 132 struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS]; 133 134 /* cid-override test input, populated by the loader before attach */ 135 __s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS]; 136 __s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS]; 137 138 /* task_ctx slab; allocated and threaded by qmap_init() */ 139 struct task_ctx __arena *task_ctxs; 140 struct task_ctx __arena *task_free_head; 141 142 /* five priority FIFOs, each a doubly-linked list through task_ctx */ 143 struct qmap_fifo fifos[5]; 144 145 /* 146 * Hierarchical sub-scheduling state. See the design comment at the top 147 * of scx_qmap.bpf.c. 148 */ 149 u32 nr_cids; /* cid count, cached at init */ 150 151 /* bpf-owned partition: read by userspace for display */ 152 struct qmap_partition part; 153 154 struct sub_sched_ctx sub_sched_ctxs[MAX_SUB_SCHEDS]; /* per-child context */ 155 u64 nr_sub_scheds; /* number of attached children */ 156 157 /* bpf-internal per-cid state */ 158 u8 cid_shared[SCX_QMAP_MAX_CPUS]; /* per cid: 1 if held shared (ENQ_IMMED-only) */ 159 160 /* allocated cid-time, charged per owner by account_alloc() */ 161 u64 alloc_ns[MAX_SUB_SCHEDS]; /* per child slot */ 162 u64 self_alloc_ns; 163 u64 alloc_ts; /* last accounting timestamp */ 164 u64 alloc_window_ns; /* total accounted time, the alloc denominator */ 165 166 /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ 167 struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ 168 struct qmap_cmask idle_cids; /* idle state of all cids regardless of delegation */ 169 struct qmap_cmask rr_cids; /* the shared pool, as a mask for grant/revoke */ 170 171 /* scratch cmasks */ 172 struct qmap_cmask to_revoke_cids; /* delta cids to revoke */ 173 struct qmap_cmask to_grant_cids; /* delta cids to grant */ 174 struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */ 175 struct qmap_cmask held_excl; /* cids held excl (ENQ): delegatable */ 176 struct qmap_cmask held_shared; /* cids held shared (ENQ_IMMED only): self-local */ 177 178 /* bpf -> userspace: stats */ 179 u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */ 180 u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */ 181 u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */ 182 u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */ 183 u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */ 184 }; 185 186 #endif /* __SCX_QMAP_H */ 187