1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2024 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMPAT_BPF_H 8 #define __SCX_COMPAT_BPF_H 9 10 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \ 11 ({ \ 12 __type __ret = 0; \ 13 if (bpf_core_enum_value_exists(__type, __ent)) \ 14 __ret = __ent; \ 15 __ret; \ 16 }) 17 18 /* v6.12: 819513666966 ("sched_ext: Add cgroup support") */ 19 struct cgroup *scx_bpf_task_cgroup___new(struct task_struct *p) __ksym __weak; 20 21 #define scx_bpf_task_cgroup(p) \ 22 (bpf_ksym_exists(scx_bpf_task_cgroup___new) ? \ 23 scx_bpf_task_cgroup___new((p)) : NULL) 24 25 /* 26 * v6.13: The verb `dispatch` was too overloaded and confusing. kfuncs are 27 * renamed to unload the verb. 28 * 29 * scx_bpf_dispatch_from_dsq() and friends were added during v6.12 by 30 * 4c30f5ce4f7a ("sched_ext: Implement scx_bpf_dispatch[_vtime]_from_dsq()"). 31 */ 32 bool scx_bpf_dsq_move_to_local___new(u64 dsq_id) __ksym __weak; 33 void scx_bpf_dsq_move_set_slice___new(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; 34 void scx_bpf_dsq_move_set_vtime___new(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; 35 bool scx_bpf_dsq_move___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 36 bool scx_bpf_dsq_move_vtime___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 37 38 bool scx_bpf_consume___old(u64 dsq_id) __ksym __weak; 39 void scx_bpf_dispatch_from_dsq_set_slice___old(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; 40 void scx_bpf_dispatch_from_dsq_set_vtime___old(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; 41 bool scx_bpf_dispatch_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 42 bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 43 44 #define scx_bpf_dsq_move_to_local(dsq_id) \ 45 (bpf_ksym_exists(scx_bpf_dsq_move_to_local___new) ? \ 46 scx_bpf_dsq_move_to_local___new((dsq_id)) : \ 47 scx_bpf_consume___old((dsq_id))) 48 49 #define scx_bpf_dsq_move_set_slice(it__iter, slice) \ 50 (bpf_ksym_exists(scx_bpf_dsq_move_set_slice___new) ? \ 51 scx_bpf_dsq_move_set_slice___new((it__iter), (slice)) : \ 52 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_slice___old) ? \ 53 scx_bpf_dispatch_from_dsq_set_slice___old((it__iter), (slice)) : \ 54 (void)0)) 55 56 #define scx_bpf_dsq_move_set_vtime(it__iter, vtime) \ 57 (bpf_ksym_exists(scx_bpf_dsq_move_set_vtime___new) ? \ 58 scx_bpf_dsq_move_set_vtime___new((it__iter), (vtime)) : \ 59 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_vtime___old) ? \ 60 scx_bpf_dispatch_from_dsq_set_vtime___old((it__iter), (vtime)) : \ 61 (void)0)) 62 63 #define scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags) \ 64 (bpf_ksym_exists(scx_bpf_dsq_move___new) ? \ 65 scx_bpf_dsq_move___new((it__iter), (p), (dsq_id), (enq_flags)) : \ 66 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq___old) ? \ 67 scx_bpf_dispatch_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \ 68 false)) 69 70 #define scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags) \ 71 (bpf_ksym_exists(scx_bpf_dsq_move_vtime___new) ? \ 72 scx_bpf_dsq_move_vtime___new((it__iter), (p), (dsq_id), (enq_flags)) : \ 73 (bpf_ksym_exists(scx_bpf_dispatch_vtime_from_dsq___old) ? \ 74 scx_bpf_dispatch_vtime_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \ 75 false)) 76 77 /* 78 * v6.15: 950ad93df2fc ("bpf: add kfunc for populating cpumask bits") 79 * 80 * Compat macro will be dropped on v6.19 release. 81 */ 82 int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; 83 84 #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ 85 (bpf_ksym_exists(bpf_cpumask_populate) ? \ 86 (bpf_cpumask_populate(cpumask, src, size__sz)) : -EOPNOTSUPP) 87 88 /* 89 * v6.19: Introduce lockless peek API for user DSQs. 90 * 91 * Preserve the following macro until v6.21. 92 */ 93 static inline struct task_struct *__COMPAT_scx_bpf_dsq_peek(u64 dsq_id) 94 { 95 struct task_struct *p = NULL; 96 struct bpf_iter_scx_dsq it; 97 98 if (bpf_ksym_exists(scx_bpf_dsq_peek)) 99 return scx_bpf_dsq_peek(dsq_id); 100 if (!bpf_iter_scx_dsq_new(&it, dsq_id, 0)) 101 p = bpf_iter_scx_dsq_next(&it); 102 bpf_iter_scx_dsq_destroy(&it); 103 return p; 104 } 105 106 /** 107 * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on 108 * in a compatible way. We will preserve this __COMPAT helper until v6.16. 109 * 110 * @enq_flags: enqueue flags from ops.enqueue() 111 * 112 * Return: True if SCX_ENQ_CPU_SELECTED is turned on in @enq_flags 113 */ 114 static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) 115 { 116 #ifdef HAVE_SCX_ENQ_CPU_SELECTED 117 /* 118 * This is the case that a BPF code compiled against vmlinux.h 119 * where the enum SCX_ENQ_CPU_SELECTED exists. 120 */ 121 122 /* 123 * We should temporarily suspend the macro expansion of 124 * 'SCX_ENQ_CPU_SELECTED'. This avoids 'SCX_ENQ_CPU_SELECTED' being 125 * rewritten to '__SCX_ENQ_CPU_SELECTED' when 'SCX_ENQ_CPU_SELECTED' 126 * is defined in 'scripts/gen_enums.py'. 127 */ 128 #pragma push_macro("SCX_ENQ_CPU_SELECTED") 129 #undef SCX_ENQ_CPU_SELECTED 130 u64 flag; 131 132 /* 133 * When the kernel did not have SCX_ENQ_CPU_SELECTED, 134 * select_task_rq_scx() has never been skipped. Thus, this case 135 * should be considered that the CPU has already been selected. 136 */ 137 if (!bpf_core_enum_value_exists(enum scx_enq_flags, 138 SCX_ENQ_CPU_SELECTED)) 139 return true; 140 141 flag = bpf_core_enum_value(enum scx_enq_flags, SCX_ENQ_CPU_SELECTED); 142 return enq_flags & flag; 143 144 /* 145 * Once done, resume the macro expansion of 'SCX_ENQ_CPU_SELECTED'. 146 */ 147 #pragma pop_macro("SCX_ENQ_CPU_SELECTED") 148 #else 149 /* 150 * This is the case that a BPF code compiled against vmlinux.h 151 * where the enum SCX_ENQ_CPU_SELECTED does NOT exist. 152 */ 153 return true; 154 #endif /* HAVE_SCX_ENQ_CPU_SELECTED */ 155 } 156 157 158 #define scx_bpf_now() \ 159 (bpf_ksym_exists(scx_bpf_now) ? \ 160 scx_bpf_now() : \ 161 bpf_ktime_get_ns()) 162 163 /* 164 * v6.15: Introduce event counters. 165 * 166 * Preserve the following macro until v6.17. 167 */ 168 #define __COMPAT_scx_bpf_events(events, size) \ 169 (bpf_ksym_exists(scx_bpf_events) ? \ 170 scx_bpf_events(events, size) : ({})) 171 172 /* 173 * v6.15: Introduce NUMA-aware kfuncs to operate with per-node idle 174 * cpumasks. 175 * 176 * Preserve the following __COMPAT_scx_*_node macros until v6.17. 177 */ 178 #define __COMPAT_scx_bpf_nr_node_ids() \ 179 (bpf_ksym_exists(scx_bpf_nr_node_ids) ? \ 180 scx_bpf_nr_node_ids() : 1U) 181 182 #define __COMPAT_scx_bpf_cpu_node(cpu) \ 183 (bpf_ksym_exists(scx_bpf_cpu_node) ? \ 184 scx_bpf_cpu_node(cpu) : 0) 185 186 #define __COMPAT_scx_bpf_get_idle_cpumask_node(node) \ 187 (bpf_ksym_exists(scx_bpf_get_idle_cpumask_node) ? \ 188 scx_bpf_get_idle_cpumask_node(node) : \ 189 scx_bpf_get_idle_cpumask()) \ 190 191 #define __COMPAT_scx_bpf_get_idle_smtmask_node(node) \ 192 (bpf_ksym_exists(scx_bpf_get_idle_smtmask_node) ? \ 193 scx_bpf_get_idle_smtmask_node(node) : \ 194 scx_bpf_get_idle_smtmask()) 195 196 #define __COMPAT_scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) \ 197 (bpf_ksym_exists(scx_bpf_pick_idle_cpu_node) ? \ 198 scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) : \ 199 scx_bpf_pick_idle_cpu(cpus_allowed, flags)) 200 201 #define __COMPAT_scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) \ 202 (bpf_ksym_exists(scx_bpf_pick_any_cpu_node) ? \ 203 scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ 204 scx_bpf_pick_any_cpu(cpus_allowed, flags)) 205 206 /* 207 * v6.18: Add a helper to retrieve the current task running on a CPU. 208 * 209 * Keep this helper available until v6.20 for compatibility. 210 */ 211 static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu) 212 { 213 struct rq *rq; 214 215 if (bpf_ksym_exists(scx_bpf_cpu_curr)) 216 return scx_bpf_cpu_curr(cpu); 217 218 rq = scx_bpf_cpu_rq(cpu); 219 220 return rq ? rq->curr : NULL; 221 } 222 223 /* 224 * v6.19: To work around BPF maximum parameter limit, the following kfuncs are 225 * replaced with variants that pack scalar arguments in a struct. Wrappers are 226 * provided to maintain source compatibility. 227 * 228 * v6.13: scx_bpf_dsq_insert_vtime() renaming is also handled here. See the 229 * block on dispatch renaming above for more details. 230 * 231 * The kernel will carry the compat variants until v6.23 to maintain binary 232 * compatibility. After v6.23 release, remove the compat handling and move the 233 * wrappers to common.bpf.h. 234 */ 235 s32 scx_bpf_select_cpu_and___compat(struct task_struct *p, s32 prev_cpu, u64 wake_flags, 236 const struct cpumask *cpus_allowed, u64 flags) __ksym __weak; 237 void scx_bpf_dispatch_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 238 void scx_bpf_dsq_insert_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 239 240 /** 241 * scx_bpf_select_cpu_and - Pick an idle CPU usable by task @p 242 * @p: task_struct to select a CPU for 243 * @prev_cpu: CPU @p was on previously 244 * @wake_flags: %SCX_WAKE_* flags 245 * @cpus_allowed: cpumask of allowed CPUs 246 * @flags: %SCX_PICK_IDLE* flags 247 * 248 * Inline wrapper that packs scalar arguments into a struct and calls 249 * __scx_bpf_select_cpu_and(). See __scx_bpf_select_cpu_and() for details. 250 */ 251 static inline s32 252 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, 253 const struct cpumask *cpus_allowed, u64 flags) 254 { 255 if (bpf_core_type_exists(struct scx_bpf_select_cpu_and_args)) { 256 struct scx_bpf_select_cpu_and_args args = { 257 .prev_cpu = prev_cpu, 258 .wake_flags = wake_flags, 259 .flags = flags, 260 }; 261 262 return __scx_bpf_select_cpu_and(p, cpus_allowed, &args); 263 } else { 264 return scx_bpf_select_cpu_and___compat(p, prev_cpu, wake_flags, 265 cpus_allowed, flags); 266 } 267 } 268 269 /* 270 * scx_bpf_select_cpu_and() is now an inline wrapper. Use this instead of 271 * bpf_ksym_exists(scx_bpf_select_cpu_and) to test availability. 272 */ 273 #define __COMPAT_HAS_scx_bpf_select_cpu_and \ 274 (bpf_core_type_exists(struct scx_bpf_select_cpu_and_args) || \ 275 bpf_ksym_exists(scx_bpf_select_cpu_and___compat)) 276 277 /** 278 * scx_bpf_dsq_insert_vtime - Insert a task into the vtime priority queue of a DSQ 279 * @p: task_struct to insert 280 * @dsq_id: DSQ to insert into 281 * @slice: duration @p can run for in nsecs, 0 to keep the current value 282 * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ 283 * @enq_flags: SCX_ENQ_* 284 * 285 * Inline wrapper that packs scalar arguments into a struct and calls 286 * __scx_bpf_dsq_insert_vtime(). See __scx_bpf_dsq_insert_vtime() for details. 287 */ 288 static inline bool 289 scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, 290 u64 enq_flags) 291 { 292 if (bpf_core_type_exists(struct scx_bpf_dsq_insert_vtime_args)) { 293 struct scx_bpf_dsq_insert_vtime_args args = { 294 .dsq_id = dsq_id, 295 .slice = slice, 296 .vtime = vtime, 297 .enq_flags = enq_flags, 298 }; 299 300 return __scx_bpf_dsq_insert_vtime(p, &args); 301 } else if (bpf_ksym_exists(scx_bpf_dsq_insert_vtime___compat)) { 302 scx_bpf_dsq_insert_vtime___compat(p, dsq_id, slice, vtime, 303 enq_flags); 304 return true; 305 } else { 306 scx_bpf_dispatch_vtime___compat(p, dsq_id, slice, vtime, 307 enq_flags); 308 return true; 309 } 310 } 311 312 /* 313 * v6.19: scx_bpf_dsq_insert() now returns bool instead of void. Move 314 * scx_bpf_dsq_insert() decl to common.bpf.h and drop compat helper after v6.22. 315 * The extra ___compat suffix is to work around libbpf not ignoring __SUFFIX on 316 * kernel side. The entire suffix can be dropped later. 317 * 318 * v6.13: scx_bpf_dsq_insert() renaming is also handled here. See the block on 319 * dispatch renaming above for more details. 320 */ 321 bool scx_bpf_dsq_insert___v2___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 322 void scx_bpf_dsq_insert___v1(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 323 void scx_bpf_dispatch___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 324 325 static inline bool 326 scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) 327 { 328 if (bpf_ksym_exists(scx_bpf_dsq_insert___v2___compat)) { 329 return scx_bpf_dsq_insert___v2___compat(p, dsq_id, slice, enq_flags); 330 } else if (bpf_ksym_exists(scx_bpf_dsq_insert___v1)) { 331 scx_bpf_dsq_insert___v1(p, dsq_id, slice, enq_flags); 332 return true; 333 } else { 334 scx_bpf_dispatch___compat(p, dsq_id, slice, enq_flags); 335 return true; 336 } 337 } 338 339 /* 340 * v6.19: scx_bpf_task_set_slice() and scx_bpf_task_set_dsq_vtime() added to for 341 * sub-sched authority checks. Drop the wrappers and move the decls to 342 * common.bpf.h after v6.22. 343 */ 344 bool scx_bpf_task_set_slice___new(struct task_struct *p, u64 slice) __ksym __weak; 345 bool scx_bpf_task_set_dsq_vtime___new(struct task_struct *p, u64 vtime) __ksym __weak; 346 347 static inline void scx_bpf_task_set_slice(struct task_struct *p, u64 slice) 348 { 349 if (bpf_ksym_exists(scx_bpf_task_set_slice___new)) 350 scx_bpf_task_set_slice___new(p, slice); 351 else 352 p->scx.slice = slice; 353 } 354 355 static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) 356 { 357 if (bpf_ksym_exists(scx_bpf_task_set_dsq_vtime___new)) 358 scx_bpf_task_set_dsq_vtime___new(p, vtime); 359 else 360 p->scx.dsq_vtime = vtime; 361 } 362 363 /* 364 * v6.19: The new void variant can be called from anywhere while the older v1 365 * variant can only be called from ops.cpu_release(). The double ___ prefixes on 366 * the v2 variant need to be removed once libbpf is updated to ignore ___ prefix 367 * on kernel side. Drop the wrapper and move the decl to common.bpf.h after 368 * v6.22. 369 */ 370 u32 scx_bpf_reenqueue_local___v1(void) __ksym __weak; 371 void scx_bpf_reenqueue_local___v2___compat(void) __ksym __weak; 372 373 static inline bool __COMPAT_scx_bpf_reenqueue_local_from_anywhere(void) 374 { 375 return bpf_ksym_exists(scx_bpf_reenqueue_local___v2___compat); 376 } 377 378 static inline void scx_bpf_reenqueue_local(void) 379 { 380 if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) 381 scx_bpf_reenqueue_local___v2___compat(); 382 else 383 scx_bpf_reenqueue_local___v1(); 384 } 385 386 /* 387 * v6.20: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This 388 * will eventually deprecate scx_bpf_reenqueue_local(). 389 */ 390 void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux__prog) __ksym __weak; 391 392 static inline bool __COMPAT_has_generic_reenq(void) 393 { 394 return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); 395 } 396 397 static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags) 398 { 399 if (bpf_ksym_exists(scx_bpf_dsq_reenq___compat)) 400 scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags, NULL); 401 else if (dsq_id == SCX_DSQ_LOCAL && reenq_flags == 0) 402 scx_bpf_reenqueue_local(); 403 else 404 scx_bpf_error("kernel too old to reenqueue foreign local or user DSQs"); 405 } 406 407 /* 408 * v7.1: %SCX_ENQ_IMMED. 409 */ 410 #define SCX_ENQ_IMMED __COMPAT_ENUM_OR_ZERO(enum scx_enq_flags, SCX_ENQ_IMMED) 411 412 /* 413 * Define sched_ext_ops. This may be expanded to define multiple variants for 414 * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH(). 415 */ 416 #define SCX_OPS_DEFINE(__name, ...) \ 417 SEC(".struct_ops.link") \ 418 struct sched_ext_ops __name = { \ 419 __VA_ARGS__, \ 420 }; 421 422 #endif /* __SCX_COMPAT_BPF_H */ 423