1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Early sched_ext type definitions. 4 * 5 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. 6 * Copyright (c) 2026 Tejun Heo <tj@kernel.org> 7 */ 8 #ifndef _KERNEL_SCHED_EXT_TYPES_H 9 #define _KERNEL_SCHED_EXT_TYPES_H 10 11 #include <linux/types.h> 12 #include <linux/jiffies.h> 13 #include <linux/overflow.h> 14 #include <linux/time64.h> 15 #include <linux/sched/topology.h> 16 17 enum scx_consts { 18 SCX_DSP_DFL_MAX_BATCH = 32, 19 SCX_DSP_MAX_LOOPS = 32, 20 SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, 21 22 /* rescue knob defaults and limits, see scx_rescue_timerfn() */ 23 SCX_RESCUE_DFL_BW_PPT = 20, /* parts per thousand, 2% */ 24 SCX_RESCUE_MAX_BW_PPT = 250, /* 25% */ 25 SCX_RESCUE_DISABLE = U32_MAX, /* disables rescue */ 26 SCX_RESCUE_DFL_QUANTUM_US = 5000, 27 SCX_RESCUE_MIN_QUANTUM_US = 1000, 28 SCX_RESCUE_MAX_QUANTUM_US = 100000, 29 SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */ 30 SCX_RESCUE_OVERLOAD_MULT = 16, /* overload threshold in funding periods */ 31 SCX_RESCUE_MIN_OVERLOAD_MS = 1000, 32 SCX_RESCUE_MAX_OVERLOAD_MS = 15000, 33 34 /* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */ 35 SCX_TID_CHUNK = 1024, 36 37 SCX_EXIT_BT_LEN = 64, 38 SCX_EXIT_MSG_LEN = 1024, 39 SCX_EXIT_DUMP_DFL_LEN = 32768, 40 41 SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, 42 43 /* 44 * Iterating all tasks may take a while. Periodically drop 45 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. 46 */ 47 SCX_TASK_ITER_BATCH = 32, 48 49 SCX_BYPASS_HOST_NTH = 2, 50 51 SCX_BYPASS_LB_DFL_INTV_US = 500 * USEC_PER_MSEC, 52 SCX_BYPASS_LB_DONOR_PCT = 125, 53 SCX_BYPASS_LB_MIN_DELTA_DIV = 4, 54 SCX_BYPASS_LB_BATCH = 256, 55 56 SCX_REENQ_MAX_REPEAT = 256, 57 58 SCX_SUB_MAX_DEPTH = 4, 59 }; 60 61 /* 62 * Per-cid topology info. For each topology level (core, cluster, LLC, node) and 63 * shard, records the first cid in the unit and its global index. Global indices 64 * are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges 65 * over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/ 66 * node fields set to -1 but always have valid shard assignments. 67 * 68 * A cluster is the cache-sharing level between the core and the LLC. Where 69 * this level is absent, each core forms a cluster of its own. Where the cache 70 * level spans the LLC, the whole LLC forms one cluster. Each cluster has a 71 * unique cluster_cid and a dense cluster_idx, and its cids form a contiguous 72 * range. 73 * 74 * Every cid therefore has a cluster, so cluster_cid alone does not say whether 75 * the machine has the level: the first cluster of an LLC starts at the LLC's 76 * own base cid and a core-wide one at the core's. 77 * 78 * Shards are contiguous CID ranges used as scalable locking/work domains for 79 * sub-scheduler operations. By default each LLC becomes one shard, split into 80 * smaller shards if the LLC exceeds the target size. No-topo cids are packed 81 * into their own max-sized shards. 82 * 83 * Shards are cut on core boundaries only, so a shard doesn't need to contain a 84 * whole cluster: a cluster may straddle two shards. Only core, cluster, LLC 85 * and node are nested, in that order. 86 * 87 * New fields are appended, never inserted: scx_bpf_cid_topo() copies this 88 * struct out sized by the program's own layout, and an older program's copy 89 * must stay a prefix of the kernel's. 90 * 91 * @core_cid: first cid of this cid's core (smt-sibling group) 92 * @core_idx: global index of that core, in [0, nr_cores_at_init) 93 * @llc_cid: first cid of this cid's LLC 94 * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init) 95 * @node_cid: first cid of this cid's NUMA node 96 * @node_idx: global index of that node, in [0, nr_nodes_at_init) 97 * @shard_cid: first cid of this cid's shard 98 * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards) 99 * @cluster_cid: first cid of this cid's cluster 100 * @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init) 101 */ 102 struct scx_cid_topo { 103 s32 core_cid; 104 s32 core_idx; 105 s32 llc_cid; 106 s32 llc_idx; 107 s32 node_cid; 108 s32 node_idx; 109 s32 shard_cid; 110 s32 shard_idx; 111 s32 cluster_cid; 112 s32 cluster_idx; 113 }; 114 115 enum scx_cid_consts { 116 SCX_CID_SHARD_SIZE_DFL = 24, 117 SCX_CID_SHARD_MAX_CPUS = 512, 118 }; 119 120 /* 121 * Per-shard metadata for O(1) shard->cid-range lookup. 122 * 123 * @base_cid: first cid of the shard 124 * @nr_cids: number of cids in the shard 125 */ 126 struct scx_cid_shard { 127 s32 base_cid; 128 s32 nr_cids; 129 }; 130 131 /* 132 * cmask: variable-length, base-windowed bitmap over cid space 133 * ----------------------------------------------------------- 134 * 135 * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the 136 * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the 137 * first (base & 63) bits of bits[0] are head padding and the trailing bits of 138 * the last active word past base + nr_cids are tail padding. Both stay zero; 139 * all mutating helpers preserve that. Words past the last active word are not 140 * read by any helper and have no constraint. 141 * 142 * Grid alignment means two cmasks always address bits[] against the same global 143 * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to 144 * 145 * dst->bits[i] OP= src->bits[i - delta] 146 * 147 * with no bit-shifting, regardless of how the two bases relate mod 64. 148 */ 149 struct scx_cmask { 150 u32 base; 151 u32 nr_cids; 152 u32 alloc_words; 153 u64 bits[]; 154 }; 155 156 /* 157 * Number of u64 words of bits[] storage that covers @nr_cids regardless of base 158 * alignment. The +1 absorbs up to 63 bits of head padding when base is not 159 * 64-aligned - always allocating one extra word beats branching on base or 160 * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids 161 * is near U32_MAX, so callers bounds-checking the result against @alloc_words 162 * catch the overflow instead of seeing a small value. 163 */ 164 #define SCX_CMASK_NR_WORDS(nr_cids) ((u32)(((u64)(nr_cids) + 63) / 64 + 1)) 165 166 /** 167 * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity 168 * @NAME: variable name to define 169 * @BASE: first cid of the active range 170 * @NR_CIDS: active range length 171 * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS 172 * 173 * @NAME aliases zero-initialized storage with the active range set to 174 * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to 175 * @ALLOC_CIDS. 176 */ 177 #define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS) \ 178 _DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \ 179 = { .base = (BASE), \ 180 .nr_cids = (NR_CIDS), \ 181 .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) }) 182 183 /** 184 * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage 185 * @NAME: variable name to define 186 * @BASE: first cid of the active range 187 * @NR_CIDS: active range length, also storage capacity 188 * 189 * @NAME aliases zero-initialized storage with the active range and storage 190 * both [BASE, BASE + NR_CIDS). 191 */ 192 #define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS) \ 193 __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS) 194 195 /** 196 * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard 197 * @NAME: variable name to define 198 * @BASE: first cid of the active range 199 * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS 200 * 201 * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by 202 * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the 203 * cmask claiming more bits than storage holds and subsequent cmask 204 * operations will overrun. 205 */ 206 #define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS) \ 207 __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS) 208 209 /* 210 * scx_cmask_ref: validated reference to a BPF-arena cmask. 211 * 212 * scx_cmask_ref_init() snapshots @base/@nr_cids. The snapshot is what 213 * downstream code uses for sizing - the live header can be mutated concurrently 214 * by BPF. 215 * 216 * scx_cmask_ref_shard() reads one shard into a cmask. scx_cmask_ref_or() and 217 * scx_cmask_ref_copy() write back into the referenced arena cmask, bounded by 218 * the snapshot. 219 * 220 * Typical input use: 221 * 222 * struct scx_cmask_ref ref; 223 * SCX_CMASK_DEFINE(shard, 0, SCX_CID_SHARD_MAX_CPUS); 224 * s32 idx, ret; 225 * 226 * ret = scx_cmask_ref_init(sch, src, &ref); 227 * if (ret < 0) 228 * return ret; 229 * 230 * for (idx = ref.shard_first; idx < ref.shard_end; idx++) { 231 * scx_cmask_ref_shard(&ref, idx, shard); 232 * if (!shard->nr_cids) 233 * continue; 234 * ... use idx and shard ... 235 * } 236 */ 237 struct scx_cmask_ref { 238 struct scx_sched *sch; 239 struct scx_cmask *src; 240 u32 base; 241 u32 nr_cids; 242 s32 shard_first; 243 s32 shard_end; 244 }; 245 246 #endif /* _KERNEL_SCHED_EXT_TYPES_H */ 247