1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2024 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMPAT_BPF_H 8 #define __SCX_COMPAT_BPF_H 9 10 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \ 11 ({ \ 12 __type __ret = 0; \ 13 if (bpf_core_enum_value_exists(__type, __ent)) \ 14 __ret = __ent; \ 15 __ret; \ 16 }) 17 18 /* v6.12: 819513666966 ("sched_ext: Add cgroup support") */ 19 #define __COMPAT_scx_bpf_task_cgroup(p) \ 20 (bpf_ksym_exists(scx_bpf_task_cgroup) ? \ 21 scx_bpf_task_cgroup((p)) : NULL) 22 23 /* 24 * v6.13: The verb `dispatch` was too overloaded and confusing. kfuncs are 25 * renamed to unload the verb. 26 * 27 * Build error is triggered if old names are used. New binaries work with both 28 * new and old names. The compat macros will be removed on v6.15 release. 29 * 30 * scx_bpf_dispatch_from_dsq() and friends were added during v6.12 by 31 * 4c30f5ce4f7a ("sched_ext: Implement scx_bpf_dispatch[_vtime]_from_dsq()"). 32 * Preserve __COMPAT macros until v6.15. 33 */ 34 void scx_bpf_dispatch___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 35 void scx_bpf_dispatch_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 36 bool scx_bpf_consume___compat(u64 dsq_id) __ksym __weak; 37 void scx_bpf_dispatch_from_dsq_set_slice___compat(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; 38 void scx_bpf_dispatch_from_dsq_set_vtime___compat(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; 39 bool scx_bpf_dispatch_from_dsq___compat(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 40 bool scx_bpf_dispatch_vtime_from_dsq___compat(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 41 int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; 42 43 #define scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags) \ 44 (bpf_ksym_exists(scx_bpf_dsq_insert) ? \ 45 scx_bpf_dsq_insert((p), (dsq_id), (slice), (enq_flags)) : \ 46 scx_bpf_dispatch___compat((p), (dsq_id), (slice), (enq_flags))) 47 48 #define scx_bpf_dsq_insert_vtime(p, dsq_id, slice, vtime, enq_flags) \ 49 (bpf_ksym_exists(scx_bpf_dsq_insert_vtime) ? \ 50 scx_bpf_dsq_insert_vtime((p), (dsq_id), (slice), (vtime), (enq_flags)) : \ 51 scx_bpf_dispatch_vtime___compat((p), (dsq_id), (slice), (vtime), (enq_flags))) 52 53 #define scx_bpf_dsq_move_to_local(dsq_id) \ 54 (bpf_ksym_exists(scx_bpf_dsq_move_to_local) ? \ 55 scx_bpf_dsq_move_to_local((dsq_id)) : \ 56 scx_bpf_consume___compat((dsq_id))) 57 58 #define __COMPAT_scx_bpf_dsq_move_set_slice(it__iter, slice) \ 59 (bpf_ksym_exists(scx_bpf_dsq_move_set_slice) ? \ 60 scx_bpf_dsq_move_set_slice((it__iter), (slice)) : \ 61 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_slice___compat) ? \ 62 scx_bpf_dispatch_from_dsq_set_slice___compat((it__iter), (slice)) : \ 63 (void)0)) 64 65 #define __COMPAT_scx_bpf_dsq_move_set_vtime(it__iter, vtime) \ 66 (bpf_ksym_exists(scx_bpf_dsq_move_set_vtime) ? \ 67 scx_bpf_dsq_move_set_vtime((it__iter), (vtime)) : \ 68 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_vtime___compat) ? \ 69 scx_bpf_dispatch_from_dsq_set_vtime___compat((it__iter), (vtime)) : \ 70 (void) 0)) 71 72 #define __COMPAT_scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags) \ 73 (bpf_ksym_exists(scx_bpf_dsq_move) ? \ 74 scx_bpf_dsq_move((it__iter), (p), (dsq_id), (enq_flags)) : \ 75 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq___compat) ? \ 76 scx_bpf_dispatch_from_dsq___compat((it__iter), (p), (dsq_id), (enq_flags)) : \ 77 false)) 78 79 #define __COMPAT_scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags) \ 80 (bpf_ksym_exists(scx_bpf_dsq_move_vtime) ? \ 81 scx_bpf_dsq_move_vtime((it__iter), (p), (dsq_id), (enq_flags)) : \ 82 (bpf_ksym_exists(scx_bpf_dispatch_vtime_from_dsq___compat) ? \ 83 scx_bpf_dispatch_vtime_from_dsq___compat((it__iter), (p), (dsq_id), (enq_flags)) : \ 84 false)) 85 86 #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ 87 (bpf_ksym_exists(bpf_cpumask_populate) ? \ 88 (bpf_cpumask_populate(cpumask, src, size__sz)) : -EOPNOTSUPP) 89 90 #define scx_bpf_dispatch(p, dsq_id, slice, enq_flags) \ 91 _Static_assert(false, "scx_bpf_dispatch() renamed to scx_bpf_dsq_insert()") 92 93 #define scx_bpf_dispatch_vtime(p, dsq_id, slice, vtime, enq_flags) \ 94 _Static_assert(false, "scx_bpf_dispatch_vtime() renamed to scx_bpf_dsq_insert_vtime()") 95 96 #define scx_bpf_consume(dsq_id) ({ \ 97 _Static_assert(false, "scx_bpf_consume() renamed to scx_bpf_dsq_move_to_local()"); \ 98 false; \ 99 }) 100 101 #define scx_bpf_dispatch_from_dsq_set_slice(it__iter, slice) \ 102 _Static_assert(false, "scx_bpf_dispatch_from_dsq_set_slice() renamed to scx_bpf_dsq_move_set_slice()") 103 104 #define scx_bpf_dispatch_from_dsq_set_vtime(it__iter, vtime) \ 105 _Static_assert(false, "scx_bpf_dispatch_from_dsq_set_vtime() renamed to scx_bpf_dsq_move_set_vtime()") 106 107 #define scx_bpf_dispatch_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ 108 _Static_assert(false, "scx_bpf_dispatch_from_dsq() renamed to scx_bpf_dsq_move()"); \ 109 false; \ 110 }) 111 112 #define scx_bpf_dispatch_vtime_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ 113 _Static_assert(false, "scx_bpf_dispatch_vtime_from_dsq() renamed to scx_bpf_dsq_move_vtime()"); \ 114 false; \ 115 }) 116 117 #define __COMPAT_scx_bpf_dispatch_from_dsq_set_slice(it__iter, slice) \ 118 _Static_assert(false, "__COMPAT_scx_bpf_dispatch_from_dsq_set_slice() renamed to __COMPAT_scx_bpf_dsq_move_set_slice()") 119 120 #define __COMPAT_scx_bpf_dispatch_from_dsq_set_vtime(it__iter, vtime) \ 121 _Static_assert(false, "__COMPAT_scx_bpf_dispatch_from_dsq_set_vtime() renamed to __COMPAT_scx_bpf_dsq_move_set_vtime()") 122 123 #define __COMPAT_scx_bpf_dispatch_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ 124 _Static_assert(false, "__COMPAT_scx_bpf_dispatch_from_dsq() renamed to __COMPAT_scx_bpf_dsq_move()"); \ 125 false; \ 126 }) 127 128 #define __COMPAT_scx_bpf_dispatch_vtime_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ 129 _Static_assert(false, "__COMPAT_scx_bpf_dispatch_vtime_from_dsq() renamed to __COMPAT_scx_bpf_dsq_move_vtime()"); \ 130 false; \ 131 }) 132 133 /** 134 * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on 135 * in a compatible way. We will preserve this __COMPAT helper until v6.16. 136 * 137 * @enq_flags: enqueue flags from ops.enqueue() 138 * 139 * Return: True if SCX_ENQ_CPU_SELECTED is turned on in @enq_flags 140 */ 141 static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) 142 { 143 #ifdef HAVE_SCX_ENQ_CPU_SELECTED 144 /* 145 * This is the case that a BPF code compiled against vmlinux.h 146 * where the enum SCX_ENQ_CPU_SELECTED exists. 147 */ 148 149 /* 150 * We should temporarily suspend the macro expansion of 151 * 'SCX_ENQ_CPU_SELECTED'. This avoids 'SCX_ENQ_CPU_SELECTED' being 152 * rewritten to '__SCX_ENQ_CPU_SELECTED' when 'SCX_ENQ_CPU_SELECTED' 153 * is defined in 'scripts/gen_enums.py'. 154 */ 155 #pragma push_macro("SCX_ENQ_CPU_SELECTED") 156 #undef SCX_ENQ_CPU_SELECTED 157 u64 flag; 158 159 /* 160 * When the kernel did not have SCX_ENQ_CPU_SELECTED, 161 * select_task_rq_scx() has never been skipped. Thus, this case 162 * should be considered that the CPU has already been selected. 163 */ 164 if (!bpf_core_enum_value_exists(enum scx_enq_flags, 165 SCX_ENQ_CPU_SELECTED)) 166 return true; 167 168 flag = bpf_core_enum_value(enum scx_enq_flags, SCX_ENQ_CPU_SELECTED); 169 return enq_flags & flag; 170 171 /* 172 * Once done, resume the macro expansion of 'SCX_ENQ_CPU_SELECTED'. 173 */ 174 #pragma pop_macro("SCX_ENQ_CPU_SELECTED") 175 #else 176 /* 177 * This is the case that a BPF code compiled against vmlinux.h 178 * where the enum SCX_ENQ_CPU_SELECTED does NOT exist. 179 */ 180 return true; 181 #endif /* HAVE_SCX_ENQ_CPU_SELECTED */ 182 } 183 184 185 #define scx_bpf_now() \ 186 (bpf_ksym_exists(scx_bpf_now) ? \ 187 scx_bpf_now() : \ 188 bpf_ktime_get_ns()) 189 190 /* 191 * v6.15: Introduce event counters. 192 * 193 * Preserve the following macro until v6.17. 194 */ 195 #define __COMPAT_scx_bpf_events(events, size) \ 196 (bpf_ksym_exists(scx_bpf_events) ? \ 197 scx_bpf_events(events, size) : ({})) 198 199 /* 200 * v6.15: Introduce NUMA-aware kfuncs to operate with per-node idle 201 * cpumasks. 202 * 203 * Preserve the following __COMPAT_scx_*_node macros until v6.17. 204 */ 205 #define __COMPAT_scx_bpf_nr_node_ids() \ 206 (bpf_ksym_exists(scx_bpf_nr_node_ids) ? \ 207 scx_bpf_nr_node_ids() : 1U) 208 209 #define __COMPAT_scx_bpf_cpu_node(cpu) \ 210 (bpf_ksym_exists(scx_bpf_cpu_node) ? \ 211 scx_bpf_cpu_node(cpu) : 0) 212 213 #define __COMPAT_scx_bpf_get_idle_cpumask_node(node) \ 214 (bpf_ksym_exists(scx_bpf_get_idle_cpumask_node) ? \ 215 scx_bpf_get_idle_cpumask_node(node) : \ 216 scx_bpf_get_idle_cpumask()) \ 217 218 #define __COMPAT_scx_bpf_get_idle_smtmask_node(node) \ 219 (bpf_ksym_exists(scx_bpf_get_idle_smtmask_node) ? \ 220 scx_bpf_get_idle_smtmask_node(node) : \ 221 scx_bpf_get_idle_smtmask()) 222 223 #define __COMPAT_scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) \ 224 (bpf_ksym_exists(scx_bpf_pick_idle_cpu_node) ? \ 225 scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) : \ 226 scx_bpf_pick_idle_cpu(cpus_allowed, flags)) 227 228 #define __COMPAT_scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) \ 229 (bpf_ksym_exists(scx_bpf_pick_any_cpu_node) ? \ 230 scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ 231 scx_bpf_pick_any_cpu(cpus_allowed, flags)) 232 233 /* 234 * v6.18: Add a helper to retrieve the current task running on a CPU. 235 * 236 * Keep this helper available until v6.20 for compatibility. 237 */ 238 static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu) 239 { 240 struct rq *rq; 241 242 if (bpf_ksym_exists(scx_bpf_cpu_curr)) 243 return scx_bpf_cpu_curr(cpu); 244 245 rq = scx_bpf_cpu_rq(cpu); 246 247 return rq ? rq->curr : NULL; 248 } 249 250 /* 251 * Define sched_ext_ops. This may be expanded to define multiple variants for 252 * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH(). 253 */ 254 #define SCX_OPS_DEFINE(__name, ...) \ 255 SEC(".struct_ops.link") \ 256 struct sched_ext_ops __name = { \ 257 __VA_ARGS__, \ 258 }; 259 260 #endif /* __SCX_COMPAT_BPF_H */ 261