1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 * Copyright (c) 2024 Tejun Heo <tj@kernel.org> 5 * Copyright (c) 2024 David Vernet <dvernet@meta.com> 6 */ 7 #ifndef __SCX_COMPAT_BPF_H 8 #define __SCX_COMPAT_BPF_H 9 10 #define __COMPAT_ENUM_OR_ZERO(__type, __ent) \ 11 ({ \ 12 __type __ret = 0; \ 13 if (bpf_core_enum_value_exists(__type, __ent)) \ 14 __ret = __ent; \ 15 __ret; \ 16 }) 17 18 /* v6.12: 819513666966 ("sched_ext: Add cgroup support") */ 19 struct cgroup *scx_bpf_task_cgroup___new(struct task_struct *p) __ksym __weak; 20 21 #define scx_bpf_task_cgroup(p) \ 22 (bpf_ksym_exists(scx_bpf_task_cgroup___new) ? \ 23 scx_bpf_task_cgroup___new((p)) : NULL) 24 25 /* 26 * v6.13: The verb `dispatch` was too overloaded and confusing. kfuncs are 27 * renamed to unload the verb. 28 * 29 * scx_bpf_dispatch_from_dsq() and friends were added during v6.12 by 30 * 4c30f5ce4f7a ("sched_ext: Implement scx_bpf_dispatch[_vtime]_from_dsq()"). 31 * 32 * v7.1: scx_bpf_dsq_move_to_local___v2() to add @enq_flags. 33 */ 34 bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags) __ksym __weak; 35 bool scx_bpf_dsq_move_to_local___v1(u64 dsq_id) __ksym __weak; 36 void scx_bpf_dsq_move_set_slice___new(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; 37 void scx_bpf_dsq_move_set_vtime___new(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; 38 bool scx_bpf_dsq_move___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 39 bool scx_bpf_dsq_move_vtime___new(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 40 41 bool scx_bpf_consume___old(u64 dsq_id) __ksym __weak; 42 void scx_bpf_dispatch_from_dsq_set_slice___old(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; 43 void scx_bpf_dispatch_from_dsq_set_vtime___old(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; 44 bool scx_bpf_dispatch_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 45 bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; 46 47 #define scx_bpf_dsq_move_to_local(dsq_id, enq_flags) \ 48 (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v2) ? \ 49 scx_bpf_dsq_move_to_local___v2((dsq_id), (enq_flags)) : \ 50 (bpf_ksym_exists(scx_bpf_dsq_move_to_local___v1) ? \ 51 scx_bpf_dsq_move_to_local___v1((dsq_id)) : \ 52 scx_bpf_consume___old((dsq_id)))) 53 54 #define scx_bpf_dsq_move_set_slice(it__iter, slice) \ 55 (bpf_ksym_exists(scx_bpf_dsq_move_set_slice___new) ? \ 56 scx_bpf_dsq_move_set_slice___new((it__iter), (slice)) : \ 57 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_slice___old) ? \ 58 scx_bpf_dispatch_from_dsq_set_slice___old((it__iter), (slice)) : \ 59 (void)0)) 60 61 #define scx_bpf_dsq_move_set_vtime(it__iter, vtime) \ 62 (bpf_ksym_exists(scx_bpf_dsq_move_set_vtime___new) ? \ 63 scx_bpf_dsq_move_set_vtime___new((it__iter), (vtime)) : \ 64 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_vtime___old) ? \ 65 scx_bpf_dispatch_from_dsq_set_vtime___old((it__iter), (vtime)) : \ 66 (void)0)) 67 68 #define scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags) \ 69 (bpf_ksym_exists(scx_bpf_dsq_move___new) ? \ 70 scx_bpf_dsq_move___new((it__iter), (p), (dsq_id), (enq_flags)) : \ 71 (bpf_ksym_exists(scx_bpf_dispatch_from_dsq___old) ? \ 72 scx_bpf_dispatch_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \ 73 false)) 74 75 #define scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags) \ 76 (bpf_ksym_exists(scx_bpf_dsq_move_vtime___new) ? \ 77 scx_bpf_dsq_move_vtime___new((it__iter), (p), (dsq_id), (enq_flags)) : \ 78 (bpf_ksym_exists(scx_bpf_dispatch_vtime_from_dsq___old) ? \ 79 scx_bpf_dispatch_vtime_from_dsq___old((it__iter), (p), (dsq_id), (enq_flags)) : \ 80 false)) 81 82 /* 83 * v6.15: 950ad93df2fc ("bpf: add kfunc for populating cpumask bits") 84 * 85 * Compat macro will be dropped on v6.19 release. 86 */ 87 int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; 88 89 #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ 90 (bpf_ksym_exists(bpf_cpumask_populate) ? \ 91 (bpf_cpumask_populate(cpumask, src, size__sz)) : -EOPNOTSUPP) 92 93 /* 94 * v6.19: Introduce lockless peek API for user DSQs. 95 * 96 * Preserve the following macro until v6.21. 97 */ 98 static inline struct task_struct *__COMPAT_scx_bpf_dsq_peek(u64 dsq_id) 99 { 100 struct task_struct *p = NULL; 101 struct bpf_iter_scx_dsq it; 102 103 if (bpf_ksym_exists(scx_bpf_dsq_peek)) 104 return scx_bpf_dsq_peek(dsq_id); 105 if (!bpf_iter_scx_dsq_new(&it, dsq_id, 0)) 106 p = bpf_iter_scx_dsq_next(&it); 107 bpf_iter_scx_dsq_destroy(&it); 108 return p; 109 } 110 111 /* 112 * v7.1: scx_bpf_sub_dispatch() for sub-sched dispatch. Preserve until 113 * we drop the compat layer for older kernels that lack the kfunc. 114 */ 115 bool scx_bpf_sub_dispatch___compat(u64 cgroup_id) __ksym __weak; 116 117 static inline bool scx_bpf_sub_dispatch(u64 cgroup_id) 118 { 119 if (bpf_ksym_exists(scx_bpf_sub_dispatch___compat)) 120 return scx_bpf_sub_dispatch___compat(cgroup_id); 121 return false; 122 } 123 124 /** 125 * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on 126 * in a compatible way. We will preserve this __COMPAT helper until v6.16. 127 * 128 * @enq_flags: enqueue flags from ops.enqueue() 129 * 130 * Return: True if SCX_ENQ_CPU_SELECTED is turned on in @enq_flags 131 */ 132 static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) 133 { 134 #ifdef HAVE_SCX_ENQ_CPU_SELECTED 135 /* 136 * This is the case that a BPF code compiled against vmlinux.h 137 * where the enum SCX_ENQ_CPU_SELECTED exists. 138 */ 139 140 /* 141 * We should temporarily suspend the macro expansion of 142 * 'SCX_ENQ_CPU_SELECTED'. This avoids 'SCX_ENQ_CPU_SELECTED' being 143 * rewritten to '__SCX_ENQ_CPU_SELECTED' when 'SCX_ENQ_CPU_SELECTED' 144 * is defined in 'scripts/gen_enums.py'. 145 */ 146 #pragma push_macro("SCX_ENQ_CPU_SELECTED") 147 #undef SCX_ENQ_CPU_SELECTED 148 u64 flag; 149 150 /* 151 * When the kernel did not have SCX_ENQ_CPU_SELECTED, 152 * select_task_rq_scx() has never been skipped. Thus, this case 153 * should be considered that the CPU has already been selected. 154 */ 155 if (!bpf_core_enum_value_exists(enum scx_enq_flags, 156 SCX_ENQ_CPU_SELECTED)) 157 return true; 158 159 flag = bpf_core_enum_value(enum scx_enq_flags, SCX_ENQ_CPU_SELECTED); 160 return enq_flags & flag; 161 162 /* 163 * Once done, resume the macro expansion of 'SCX_ENQ_CPU_SELECTED'. 164 */ 165 #pragma pop_macro("SCX_ENQ_CPU_SELECTED") 166 #else 167 /* 168 * This is the case that a BPF code compiled against vmlinux.h 169 * where the enum SCX_ENQ_CPU_SELECTED does NOT exist. 170 */ 171 return true; 172 #endif /* HAVE_SCX_ENQ_CPU_SELECTED */ 173 } 174 175 176 #define scx_bpf_now() \ 177 (bpf_ksym_exists(scx_bpf_now) ? \ 178 scx_bpf_now() : \ 179 bpf_ktime_get_ns()) 180 181 /* 182 * v6.15: Introduce event counters. 183 * 184 * Preserve the following macro until v6.17. 185 */ 186 #define __COMPAT_scx_bpf_events(events, size) \ 187 (bpf_ksym_exists(scx_bpf_events) ? \ 188 scx_bpf_events(events, size) : ({})) 189 190 /* 191 * v6.15: Introduce NUMA-aware kfuncs to operate with per-node idle 192 * cpumasks. 193 * 194 * Preserve the following __COMPAT_scx_*_node macros until v6.17. 195 */ 196 #define __COMPAT_scx_bpf_nr_node_ids() \ 197 (bpf_ksym_exists(scx_bpf_nr_node_ids) ? \ 198 scx_bpf_nr_node_ids() : 1U) 199 200 #define __COMPAT_scx_bpf_cpu_node(cpu) \ 201 (bpf_ksym_exists(scx_bpf_cpu_node) ? \ 202 scx_bpf_cpu_node(cpu) : 0) 203 204 #define __COMPAT_scx_bpf_get_idle_cpumask_node(node) \ 205 (bpf_ksym_exists(scx_bpf_get_idle_cpumask_node) ? \ 206 scx_bpf_get_idle_cpumask_node(node) : \ 207 scx_bpf_get_idle_cpumask()) \ 208 209 #define __COMPAT_scx_bpf_get_idle_smtmask_node(node) \ 210 (bpf_ksym_exists(scx_bpf_get_idle_smtmask_node) ? \ 211 scx_bpf_get_idle_smtmask_node(node) : \ 212 scx_bpf_get_idle_smtmask()) 213 214 #define __COMPAT_scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) \ 215 (bpf_ksym_exists(scx_bpf_pick_idle_cpu_node) ? \ 216 scx_bpf_pick_idle_cpu_node(cpus_allowed, node, flags) : \ 217 scx_bpf_pick_idle_cpu(cpus_allowed, flags)) 218 219 #define __COMPAT_scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) \ 220 (bpf_ksym_exists(scx_bpf_pick_any_cpu_node) ? \ 221 scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ 222 scx_bpf_pick_any_cpu(cpus_allowed, flags)) 223 224 /* 225 * v6.18: Add a helper to retrieve the current task running on a CPU. 226 * 227 * Keep this helper available until v6.20 for compatibility. 228 */ 229 static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu) 230 { 231 struct rq *rq; 232 233 if (bpf_ksym_exists(scx_bpf_cpu_curr)) 234 return scx_bpf_cpu_curr(cpu); 235 236 rq = scx_bpf_cpu_rq(cpu); 237 238 return rq ? rq->curr : NULL; 239 } 240 241 /* 242 * v6.19: To work around BPF maximum parameter limit, the following kfuncs are 243 * replaced with variants that pack scalar arguments in a struct. Wrappers are 244 * provided to maintain source compatibility. 245 * 246 * v6.13: scx_bpf_dsq_insert_vtime() renaming is also handled here. See the 247 * block on dispatch renaming above for more details. 248 * 249 * The kernel will carry the compat variants until v6.23 to maintain binary 250 * compatibility. After v6.23 release, remove the compat handling and move the 251 * wrappers to common.bpf.h. 252 */ 253 s32 scx_bpf_select_cpu_and___compat(struct task_struct *p, s32 prev_cpu, u64 wake_flags, 254 const struct cpumask *cpus_allowed, u64 flags) __ksym __weak; 255 void scx_bpf_dispatch_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 256 void scx_bpf_dsq_insert_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; 257 258 /** 259 * scx_bpf_select_cpu_and - Pick an idle CPU usable by task @p 260 * @p: task_struct to select a CPU for 261 * @prev_cpu: CPU @p was on previously 262 * @wake_flags: %SCX_WAKE_* flags 263 * @cpus_allowed: cpumask of allowed CPUs 264 * @flags: %SCX_PICK_IDLE* flags 265 * 266 * Inline wrapper that packs scalar arguments into a struct and calls 267 * __scx_bpf_select_cpu_and(). See __scx_bpf_select_cpu_and() for details. 268 */ 269 static inline s32 270 scx_bpf_select_cpu_and(struct task_struct *p, s32 prev_cpu, u64 wake_flags, 271 const struct cpumask *cpus_allowed, u64 flags) 272 { 273 if (bpf_core_type_exists(struct scx_bpf_select_cpu_and_args)) { 274 struct scx_bpf_select_cpu_and_args args = { 275 .prev_cpu = prev_cpu, 276 .wake_flags = wake_flags, 277 .flags = flags, 278 }; 279 280 return __scx_bpf_select_cpu_and(p, cpus_allowed, &args); 281 } else { 282 return scx_bpf_select_cpu_and___compat(p, prev_cpu, wake_flags, 283 cpus_allowed, flags); 284 } 285 } 286 287 /* 288 * scx_bpf_select_cpu_and() is now an inline wrapper. Use this instead of 289 * bpf_ksym_exists(scx_bpf_select_cpu_and) to test availability. 290 */ 291 #define __COMPAT_HAS_scx_bpf_select_cpu_and \ 292 (bpf_core_type_exists(struct scx_bpf_select_cpu_and_args) || \ 293 bpf_ksym_exists(scx_bpf_select_cpu_and___compat)) 294 295 /** 296 * scx_bpf_dsq_insert_vtime - Insert a task into the vtime priority queue of a DSQ 297 * @p: task_struct to insert 298 * @dsq_id: DSQ to insert into 299 * @slice: duration @p can run for in nsecs, 0 to keep the current value 300 * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ 301 * @enq_flags: SCX_ENQ_* 302 * 303 * Inline wrapper that packs scalar arguments into a struct and calls 304 * __scx_bpf_dsq_insert_vtime(). See __scx_bpf_dsq_insert_vtime() for details. 305 */ 306 static inline bool 307 scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, 308 u64 enq_flags) 309 { 310 if (bpf_core_type_exists(struct scx_bpf_dsq_insert_vtime_args)) { 311 struct scx_bpf_dsq_insert_vtime_args args = { 312 .dsq_id = dsq_id, 313 .slice = slice, 314 .vtime = vtime, 315 .enq_flags = enq_flags, 316 }; 317 318 return __scx_bpf_dsq_insert_vtime(p, &args); 319 } else if (bpf_ksym_exists(scx_bpf_dsq_insert_vtime___compat)) { 320 scx_bpf_dsq_insert_vtime___compat(p, dsq_id, slice, vtime, 321 enq_flags); 322 return true; 323 } else { 324 scx_bpf_dispatch_vtime___compat(p, dsq_id, slice, vtime, 325 enq_flags); 326 return true; 327 } 328 } 329 330 /* 331 * v6.19: scx_bpf_dsq_insert() now returns bool instead of void. Move 332 * scx_bpf_dsq_insert() decl to common.bpf.h and drop compat helper after v6.22. 333 * The extra ___compat suffix is to work around libbpf not ignoring __SUFFIX on 334 * kernel side. The entire suffix can be dropped later. 335 * 336 * v6.13: scx_bpf_dsq_insert() renaming is also handled here. See the block on 337 * dispatch renaming above for more details. 338 */ 339 bool scx_bpf_dsq_insert___v2___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 340 void scx_bpf_dsq_insert___v1(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 341 void scx_bpf_dispatch___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; 342 343 static inline bool 344 scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) 345 { 346 if (bpf_ksym_exists(scx_bpf_dsq_insert___v2___compat)) { 347 return scx_bpf_dsq_insert___v2___compat(p, dsq_id, slice, enq_flags); 348 } else if (bpf_ksym_exists(scx_bpf_dsq_insert___v1)) { 349 scx_bpf_dsq_insert___v1(p, dsq_id, slice, enq_flags); 350 return true; 351 } else { 352 scx_bpf_dispatch___compat(p, dsq_id, slice, enq_flags); 353 return true; 354 } 355 } 356 357 /* 358 * v6.19: scx_bpf_task_set_slice() and scx_bpf_task_set_dsq_vtime() added to for 359 * sub-sched authority checks. Drop the wrappers and move the decls to 360 * common.bpf.h after v6.22. 361 */ 362 bool scx_bpf_task_set_slice___new(struct task_struct *p, u64 slice) __ksym __weak; 363 bool scx_bpf_task_set_dsq_vtime___new(struct task_struct *p, u64 vtime) __ksym __weak; 364 365 static inline void scx_bpf_task_set_slice(struct task_struct *p, u64 slice) 366 { 367 if (bpf_ksym_exists(scx_bpf_task_set_slice___new)) 368 scx_bpf_task_set_slice___new(p, slice); 369 else 370 p->scx.slice = slice; 371 } 372 373 static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) 374 { 375 if (bpf_ksym_exists(scx_bpf_task_set_dsq_vtime___new)) 376 scx_bpf_task_set_dsq_vtime___new(p, vtime); 377 else 378 p->scx.dsq_vtime = vtime; 379 } 380 381 /* 382 * v6.19: The new void variant can be called from anywhere while the older v1 383 * variant can only be called from ops.cpu_release(). The double ___ prefixes on 384 * the v2 variant need to be removed once libbpf is updated to ignore ___ prefix 385 * on kernel side. Drop the wrapper and move the decl to common.bpf.h after 386 * v6.22. 387 */ 388 u32 scx_bpf_reenqueue_local___v1(void) __ksym __weak; 389 void scx_bpf_reenqueue_local___v2___compat(void) __ksym __weak; 390 391 static inline bool __COMPAT_scx_bpf_reenqueue_local_from_anywhere(void) 392 { 393 return bpf_ksym_exists(scx_bpf_reenqueue_local___v2___compat); 394 } 395 396 static inline void scx_bpf_reenqueue_local(void) 397 { 398 if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) 399 scx_bpf_reenqueue_local___v2___compat(); 400 else 401 scx_bpf_reenqueue_local___v1(); 402 } 403 404 /* 405 * v6.20: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This 406 * will eventually deprecate scx_bpf_reenqueue_local(). 407 */ 408 void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags, const struct bpf_prog_aux *aux__prog) __ksym __weak; 409 410 static inline bool __COMPAT_has_generic_reenq(void) 411 { 412 return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); 413 } 414 415 static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags) 416 { 417 if (bpf_ksym_exists(scx_bpf_dsq_reenq___compat)) 418 scx_bpf_dsq_reenq___compat(dsq_id, reenq_flags, NULL); 419 else if (dsq_id == SCX_DSQ_LOCAL && reenq_flags == 0) 420 scx_bpf_reenqueue_local(); 421 else 422 scx_bpf_error("kernel too old to reenqueue foreign local or user DSQs"); 423 } 424 425 /* 426 * Define sched_ext_ops. This may be expanded to define multiple variants for 427 * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH(). 428 */ 429 #define SCX_OPS_DEFINE(__name, ...) \ 430 SEC(".struct_ops.link") \ 431 struct sched_ext_ops __name = { \ 432 __VA_ARGS__, \ 433 }; 434 435 #endif /* __SCX_COMPAT_BPF_H */ 436