1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. 6 * Copyright (c) 2025 Tejun Heo <tj@kernel.org> 7 */ 8 #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) 9 10 enum scx_consts { 11 SCX_DSP_DFL_MAX_BATCH = 32, 12 SCX_DSP_MAX_LOOPS = 32, 13 SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, 14 15 SCX_EXIT_BT_LEN = 64, 16 SCX_EXIT_MSG_LEN = 1024, 17 SCX_EXIT_DUMP_DFL_LEN = 32768, 18 19 SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, 20 21 /* 22 * Iterating all tasks may take a while. Periodically drop 23 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. 24 */ 25 SCX_TASK_ITER_BATCH = 32, 26 }; 27 28 enum scx_exit_kind { 29 SCX_EXIT_NONE, 30 SCX_EXIT_DONE, 31 32 SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ 33 SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ 34 SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ 35 SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ 36 37 SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ 38 SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ 39 SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ 40 }; 41 42 /* 43 * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), 44 * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes 45 * are 64bit of the format: 46 * 47 * Bits: [63 .. 48 47 .. 32 31 .. 0] 48 * [ SYS ACT ] [ SYS RSN ] [ USR ] 49 * 50 * SYS ACT: System-defined exit actions 51 * SYS RSN: System-defined exit reasons 52 * USR : User-defined exit codes and reasons 53 * 54 * Using the above, users may communicate intention and context by ORing system 55 * actions and/or system reasons with a user-defined exit code. 56 */ 57 enum scx_exit_code { 58 /* Reasons */ 59 SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, 60 61 /* Actions */ 62 SCX_ECODE_ACT_RESTART = 1LLU << 48, 63 }; 64 65 /* 66 * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is 67 * being disabled. 68 */ 69 struct scx_exit_info { 70 /* %SCX_EXIT_* - broad category of the exit reason */ 71 enum scx_exit_kind kind; 72 73 /* exit code if gracefully exiting */ 74 s64 exit_code; 75 76 /* textual representation of the above */ 77 const char *reason; 78 79 /* backtrace if exiting due to an error */ 80 unsigned long *bt; 81 u32 bt_len; 82 83 /* informational message */ 84 char *msg; 85 86 /* debug dump */ 87 char *dump; 88 }; 89 90 /* sched_ext_ops.flags */ 91 enum scx_ops_flags { 92 /* 93 * Keep built-in idle tracking even if ops.update_idle() is implemented. 94 */ 95 SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, 96 97 /* 98 * By default, if there are no other task to run on the CPU, ext core 99 * keeps running the current task even after its slice expires. If this 100 * flag is specified, such tasks are passed to ops.enqueue() with 101 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. 102 */ 103 SCX_OPS_ENQ_LAST = 1LLU << 1, 104 105 /* 106 * An exiting task may schedule after PF_EXITING is set. In such cases, 107 * bpf_task_from_pid() may not be able to find the task and if the BPF 108 * scheduler depends on pid lookup for dispatching, the task will be 109 * lost leading to various issues including RCU grace period stalls. 110 * 111 * To mask this problem, by default, unhashed tasks are automatically 112 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't 113 * depend on pid lookups and wants to handle these tasks directly, the 114 * following flag can be used. 115 */ 116 SCX_OPS_ENQ_EXITING = 1LLU << 2, 117 118 /* 119 * If set, only tasks with policy set to SCHED_EXT are attached to 120 * sched_ext. If clear, SCHED_NORMAL tasks are also included. 121 */ 122 SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, 123 124 /* 125 * A migration disabled task can only execute on its current CPU. By 126 * default, such tasks are automatically put on the CPU's local DSQ with 127 * the default slice on enqueue. If this ops flag is set, they also go 128 * through ops.enqueue(). 129 * 130 * A migration disabled task never invokes ops.select_cpu() as it can 131 * only select the current CPU. Also, p->cpus_ptr will only contain its 132 * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr 133 * and thus may disagree with cpumask_weight(p->cpus_ptr). 134 */ 135 SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, 136 137 /* 138 * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes 139 * ops.enqueue() on the ops.select_cpu() selected or the wakee's 140 * previous CPU via IPI (inter-processor interrupt) to reduce cacheline 141 * transfers. When this optimization is enabled, ops.select_cpu() is 142 * skipped in some cases (when racing against the wakee switching out). 143 * As the BPF scheduler may depend on ops.select_cpu() being invoked 144 * during wakeups, queued wakeup is disabled by default. 145 * 146 * If this ops flag is set, queued wakeup optimization is enabled and 147 * the BPF scheduler must be able to handle ops.enqueue() invoked on the 148 * wakee's CPU without preceding ops.select_cpu() even for tasks which 149 * may be executed on multiple CPUs. 150 */ 151 SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, 152 153 /* 154 * If set, enable per-node idle cpumasks. If clear, use a single global 155 * flat idle cpumask. 156 */ 157 SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, 158 159 /* 160 * CPU cgroup support flags 161 */ 162 SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16, /* DEPRECATED, will be removed on 6.18 */ 163 164 SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | 165 SCX_OPS_ENQ_LAST | 166 SCX_OPS_ENQ_EXITING | 167 SCX_OPS_ENQ_MIGRATION_DISABLED | 168 SCX_OPS_ALLOW_QUEUED_WAKEUP | 169 SCX_OPS_SWITCH_PARTIAL | 170 SCX_OPS_BUILTIN_IDLE_PER_NODE | 171 SCX_OPS_HAS_CGROUP_WEIGHT, 172 173 /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ 174 __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, 175 176 SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, 177 }; 178 179 /* argument container for ops.init_task() */ 180 struct scx_init_task_args { 181 /* 182 * Set if ops.init_task() is being invoked on the fork path, as opposed 183 * to the scheduler transition path. 184 */ 185 bool fork; 186 #ifdef CONFIG_EXT_GROUP_SCHED 187 /* the cgroup the task is joining */ 188 struct cgroup *cgroup; 189 #endif 190 }; 191 192 /* argument container for ops.exit_task() */ 193 struct scx_exit_task_args { 194 /* Whether the task exited before running on sched_ext. */ 195 bool cancelled; 196 }; 197 198 /* argument container for ops->cgroup_init() */ 199 struct scx_cgroup_init_args { 200 /* the weight of the cgroup [1..10000] */ 201 u32 weight; 202 203 /* bandwidth control parameters from cpu.max and cpu.max.burst */ 204 u64 bw_period_us; 205 u64 bw_quota_us; 206 u64 bw_burst_us; 207 }; 208 209 enum scx_cpu_preempt_reason { 210 /* next task is being scheduled by &sched_class_rt */ 211 SCX_CPU_PREEMPT_RT, 212 /* next task is being scheduled by &sched_class_dl */ 213 SCX_CPU_PREEMPT_DL, 214 /* next task is being scheduled by &sched_class_stop */ 215 SCX_CPU_PREEMPT_STOP, 216 /* unknown reason for SCX being preempted */ 217 SCX_CPU_PREEMPT_UNKNOWN, 218 }; 219 220 /* 221 * Argument container for ops->cpu_acquire(). Currently empty, but may be 222 * expanded in the future. 223 */ 224 struct scx_cpu_acquire_args {}; 225 226 /* argument container for ops->cpu_release() */ 227 struct scx_cpu_release_args { 228 /* the reason the CPU was preempted */ 229 enum scx_cpu_preempt_reason reason; 230 231 /* the task that's going to be scheduled on the CPU */ 232 struct task_struct *task; 233 }; 234 235 /* 236 * Informational context provided to dump operations. 237 */ 238 struct scx_dump_ctx { 239 enum scx_exit_kind kind; 240 s64 exit_code; 241 const char *reason; 242 u64 at_ns; 243 u64 at_jiffies; 244 }; 245 246 /** 247 * struct sched_ext_ops - Operation table for BPF scheduler implementation 248 * 249 * A BPF scheduler can implement an arbitrary scheduling policy by 250 * implementing and loading operations in this table. Note that a userland 251 * scheduling policy can also be implemented using the BPF scheduler 252 * as a shim layer. 253 */ 254 struct sched_ext_ops { 255 /** 256 * @select_cpu: Pick the target CPU for a task which is being woken up 257 * @p: task being woken up 258 * @prev_cpu: the cpu @p was on before sleeping 259 * @wake_flags: SCX_WAKE_* 260 * 261 * Decision made here isn't final. @p may be moved to any CPU while it 262 * is getting dispatched for execution later. However, as @p is not on 263 * the rq at this point, getting the eventual execution CPU right here 264 * saves a small bit of overhead down the line. 265 * 266 * If an idle CPU is returned, the CPU is kicked and will try to 267 * dispatch. While an explicit custom mechanism can be added, 268 * select_cpu() serves as the default way to wake up idle CPUs. 269 * 270 * @p may be inserted into a DSQ directly by calling 271 * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. 272 * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ 273 * of the CPU returned by this operation. 274 * 275 * Note that select_cpu() is never called for tasks that can only run 276 * on a single CPU or tasks with migration disabled, as they don't have 277 * the option to select a different CPU. See select_task_rq() for 278 * details. 279 */ 280 s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 281 282 /** 283 * @enqueue: Enqueue a task on the BPF scheduler 284 * @p: task being enqueued 285 * @enq_flags: %SCX_ENQ_* 286 * 287 * @p is ready to run. Insert directly into a DSQ by calling 288 * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly 289 * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, 290 * the task will stall. 291 * 292 * If @p was inserted into a DSQ from ops.select_cpu(), this callback is 293 * skipped. 294 */ 295 void (*enqueue)(struct task_struct *p, u64 enq_flags); 296 297 /** 298 * @dequeue: Remove a task from the BPF scheduler 299 * @p: task being dequeued 300 * @deq_flags: %SCX_DEQ_* 301 * 302 * Remove @p from the BPF scheduler. This is usually called to isolate 303 * the task while updating its scheduling properties (e.g. priority). 304 * 305 * The ext core keeps track of whether the BPF side owns a given task or 306 * not and can gracefully ignore spurious dispatches from BPF side, 307 * which makes it safe to not implement this method. However, depending 308 * on the scheduling logic, this can lead to confusing behaviors - e.g. 309 * scheduling position not being updated across a priority change. 310 */ 311 void (*dequeue)(struct task_struct *p, u64 deq_flags); 312 313 /** 314 * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs 315 * @cpu: CPU to dispatch tasks for 316 * @prev: previous task being switched out 317 * 318 * Called when a CPU's local dsq is empty. The operation should dispatch 319 * one or more tasks from the BPF scheduler into the DSQs using 320 * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ 321 * using scx_bpf_dsq_move_to_local(). 322 * 323 * The maximum number of times scx_bpf_dsq_insert() can be called 324 * without an intervening scx_bpf_dsq_move_to_local() is specified by 325 * ops.dispatch_max_batch. See the comments on top of the two functions 326 * for more details. 327 * 328 * When not %NULL, @prev is an SCX task with its slice depleted. If 329 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in 330 * @prev->scx.flags, it is not enqueued yet and will be enqueued after 331 * ops.dispatch() returns. To keep executing @prev, return without 332 * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. 333 */ 334 void (*dispatch)(s32 cpu, struct task_struct *prev); 335 336 /** 337 * @tick: Periodic tick 338 * @p: task running currently 339 * 340 * This operation is called every 1/HZ seconds on CPUs which are 341 * executing an SCX task. Setting @p->scx.slice to 0 will trigger an 342 * immediate dispatch cycle on the CPU. 343 */ 344 void (*tick)(struct task_struct *p); 345 346 /** 347 * @runnable: A task is becoming runnable on its associated CPU 348 * @p: task becoming runnable 349 * @enq_flags: %SCX_ENQ_* 350 * 351 * This and the following three functions can be used to track a task's 352 * execution state transitions. A task becomes ->runnable() on a CPU, 353 * and then goes through one or more ->running() and ->stopping() pairs 354 * as it runs on the CPU, and eventually becomes ->quiescent() when it's 355 * done running on the CPU. 356 * 357 * @p is becoming runnable on the CPU because it's 358 * 359 * - waking up (%SCX_ENQ_WAKEUP) 360 * - being moved from another CPU 361 * - being restored after temporarily taken off the queue for an 362 * attribute change. 363 * 364 * This and ->enqueue() are related but not coupled. This operation 365 * notifies @p's state transition and may not be followed by ->enqueue() 366 * e.g. when @p is being dispatched to a remote CPU, or when @p is 367 * being enqueued on a CPU experiencing a hotplug event. Likewise, a 368 * task may be ->enqueue()'d without being preceded by this operation 369 * e.g. after exhausting its slice. 370 */ 371 void (*runnable)(struct task_struct *p, u64 enq_flags); 372 373 /** 374 * @running: A task is starting to run on its associated CPU 375 * @p: task starting to run 376 * 377 * Note that this callback may be called from a CPU other than the 378 * one the task is going to run on. This can happen when a task 379 * property is changed (i.e., affinity), since scx_next_task_scx(), 380 * which triggers this callback, may run on a CPU different from 381 * the task's assigned CPU. 382 * 383 * Therefore, always use scx_bpf_task_cpu(@p) to determine the 384 * target CPU the task is going to use. 385 * 386 * See ->runnable() for explanation on the task state notifiers. 387 */ 388 void (*running)(struct task_struct *p); 389 390 /** 391 * @stopping: A task is stopping execution 392 * @p: task stopping to run 393 * @runnable: is task @p still runnable? 394 * 395 * Note that this callback may be called from a CPU other than the 396 * one the task was running on. This can happen when a task 397 * property is changed (i.e., affinity), since dequeue_task_scx(), 398 * which triggers this callback, may run on a CPU different from 399 * the task's assigned CPU. 400 * 401 * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU 402 * the task was running on. 403 * 404 * See ->runnable() for explanation on the task state notifiers. If 405 * !@runnable, ->quiescent() will be invoked after this operation 406 * returns. 407 */ 408 void (*stopping)(struct task_struct *p, bool runnable); 409 410 /** 411 * @quiescent: A task is becoming not runnable on its associated CPU 412 * @p: task becoming not runnable 413 * @deq_flags: %SCX_DEQ_* 414 * 415 * See ->runnable() for explanation on the task state notifiers. 416 * 417 * @p is becoming quiescent on the CPU because it's 418 * 419 * - sleeping (%SCX_DEQ_SLEEP) 420 * - being moved to another CPU 421 * - being temporarily taken off the queue for an attribute change 422 * (%SCX_DEQ_SAVE) 423 * 424 * This and ->dequeue() are related but not coupled. This operation 425 * notifies @p's state transition and may not be preceded by ->dequeue() 426 * e.g. when @p is being dispatched to a remote CPU. 427 */ 428 void (*quiescent)(struct task_struct *p, u64 deq_flags); 429 430 /** 431 * @yield: Yield CPU 432 * @from: yielding task 433 * @to: optional yield target task 434 * 435 * If @to is NULL, @from is yielding the CPU to other runnable tasks. 436 * The BPF scheduler should ensure that other available tasks are 437 * dispatched before the yielding task. Return value is ignored in this 438 * case. 439 * 440 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf 441 * scheduler can implement the request, return %true; otherwise, %false. 442 */ 443 bool (*yield)(struct task_struct *from, struct task_struct *to); 444 445 /** 446 * @core_sched_before: Task ordering for core-sched 447 * @a: task A 448 * @b: task B 449 * 450 * Used by core-sched to determine the ordering between two tasks. See 451 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on 452 * core-sched. 453 * 454 * Both @a and @b are runnable and may or may not currently be queued on 455 * the BPF scheduler. Should return %true if @a should run before @b. 456 * %false if there's no required ordering or @b should run before @a. 457 * 458 * If not specified, the default is ordering them according to when they 459 * became runnable. 460 */ 461 bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); 462 463 /** 464 * @set_weight: Set task weight 465 * @p: task to set weight for 466 * @weight: new weight [1..10000] 467 * 468 * Update @p's weight to @weight. 469 */ 470 void (*set_weight)(struct task_struct *p, u32 weight); 471 472 /** 473 * @set_cpumask: Set CPU affinity 474 * @p: task to set CPU affinity for 475 * @cpumask: cpumask of cpus that @p can run on 476 * 477 * Update @p's CPU affinity to @cpumask. 478 */ 479 void (*set_cpumask)(struct task_struct *p, 480 const struct cpumask *cpumask); 481 482 /** 483 * @update_idle: Update the idle state of a CPU 484 * @cpu: CPU to update the idle state for 485 * @idle: whether entering or exiting the idle state 486 * 487 * This operation is called when @rq's CPU goes or leaves the idle 488 * state. By default, implementing this operation disables the built-in 489 * idle CPU tracking and the following helpers become unavailable: 490 * 491 * - scx_bpf_select_cpu_dfl() 492 * - scx_bpf_select_cpu_and() 493 * - scx_bpf_test_and_clear_cpu_idle() 494 * - scx_bpf_pick_idle_cpu() 495 * 496 * The user also must implement ops.select_cpu() as the default 497 * implementation relies on scx_bpf_select_cpu_dfl(). 498 * 499 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle 500 * tracking. 501 */ 502 void (*update_idle)(s32 cpu, bool idle); 503 504 /** 505 * @cpu_acquire: A CPU is becoming available to the BPF scheduler 506 * @cpu: The CPU being acquired by the BPF scheduler. 507 * @args: Acquire arguments, see the struct definition. 508 * 509 * A CPU that was previously released from the BPF scheduler is now once 510 * again under its control. 511 */ 512 void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); 513 514 /** 515 * @cpu_release: A CPU is taken away from the BPF scheduler 516 * @cpu: The CPU being released by the BPF scheduler. 517 * @args: Release arguments, see the struct definition. 518 * 519 * The specified CPU is no longer under the control of the BPF 520 * scheduler. This could be because it was preempted by a higher 521 * priority sched_class, though there may be other reasons as well. The 522 * caller should consult @args->reason to determine the cause. 523 */ 524 void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); 525 526 /** 527 * @init_task: Initialize a task to run in a BPF scheduler 528 * @p: task to initialize for BPF scheduling 529 * @args: init arguments, see the struct definition 530 * 531 * Either we're loading a BPF scheduler or a new task is being forked. 532 * Initialize @p for BPF scheduling. This operation may block and can 533 * be used for allocations, and is called exactly once for a task. 534 * 535 * Return 0 for success, -errno for failure. An error return while 536 * loading will abort loading of the BPF scheduler. During a fork, it 537 * will abort that specific fork. 538 */ 539 s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); 540 541 /** 542 * @exit_task: Exit a previously-running task from the system 543 * @p: task to exit 544 * @args: exit arguments, see the struct definition 545 * 546 * @p is exiting or the BPF scheduler is being unloaded. Perform any 547 * necessary cleanup for @p. 548 */ 549 void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); 550 551 /** 552 * @enable: Enable BPF scheduling for a task 553 * @p: task to enable BPF scheduling for 554 * 555 * Enable @p for BPF scheduling. enable() is called on @p any time it 556 * enters SCX, and is always paired with a matching disable(). 557 */ 558 void (*enable)(struct task_struct *p); 559 560 /** 561 * @disable: Disable BPF scheduling for a task 562 * @p: task to disable BPF scheduling for 563 * 564 * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. 565 * Disable BPF scheduling for @p. A disable() call is always matched 566 * with a prior enable() call. 567 */ 568 void (*disable)(struct task_struct *p); 569 570 /** 571 * @dump: Dump BPF scheduler state on error 572 * @ctx: debug dump context 573 * 574 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. 575 */ 576 void (*dump)(struct scx_dump_ctx *ctx); 577 578 /** 579 * @dump_cpu: Dump BPF scheduler state for a CPU on error 580 * @ctx: debug dump context 581 * @cpu: CPU to generate debug dump for 582 * @idle: @cpu is currently idle without any runnable tasks 583 * 584 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 585 * @cpu. If @idle is %true and this operation doesn't produce any 586 * output, @cpu is skipped for dump. 587 */ 588 void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); 589 590 /** 591 * @dump_task: Dump BPF scheduler state for a runnable task on error 592 * @ctx: debug dump context 593 * @p: runnable task to generate debug dump for 594 * 595 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 596 * @p. 597 */ 598 void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); 599 600 #ifdef CONFIG_EXT_GROUP_SCHED 601 /** 602 * @cgroup_init: Initialize a cgroup 603 * @cgrp: cgroup being initialized 604 * @args: init arguments, see the struct definition 605 * 606 * Either the BPF scheduler is being loaded or @cgrp created, initialize 607 * @cgrp for sched_ext. This operation may block. 608 * 609 * Return 0 for success, -errno for failure. An error return while 610 * loading will abort loading of the BPF scheduler. During cgroup 611 * creation, it will abort the specific cgroup creation. 612 */ 613 s32 (*cgroup_init)(struct cgroup *cgrp, 614 struct scx_cgroup_init_args *args); 615 616 /** 617 * @cgroup_exit: Exit a cgroup 618 * @cgrp: cgroup being exited 619 * 620 * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit 621 * @cgrp for sched_ext. This operation my block. 622 */ 623 void (*cgroup_exit)(struct cgroup *cgrp); 624 625 /** 626 * @cgroup_prep_move: Prepare a task to be moved to a different cgroup 627 * @p: task being moved 628 * @from: cgroup @p is being moved from 629 * @to: cgroup @p is being moved to 630 * 631 * Prepare @p for move from cgroup @from to @to. This operation may 632 * block and can be used for allocations. 633 * 634 * Return 0 for success, -errno for failure. An error return aborts the 635 * migration. 636 */ 637 s32 (*cgroup_prep_move)(struct task_struct *p, 638 struct cgroup *from, struct cgroup *to); 639 640 /** 641 * @cgroup_move: Commit cgroup move 642 * @p: task being moved 643 * @from: cgroup @p is being moved from 644 * @to: cgroup @p is being moved to 645 * 646 * Commit the move. @p is dequeued during this operation. 647 */ 648 void (*cgroup_move)(struct task_struct *p, 649 struct cgroup *from, struct cgroup *to); 650 651 /** 652 * @cgroup_cancel_move: Cancel cgroup move 653 * @p: task whose cgroup move is being canceled 654 * @from: cgroup @p was being moved from 655 * @to: cgroup @p was being moved to 656 * 657 * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). 658 * Undo the preparation. 659 */ 660 void (*cgroup_cancel_move)(struct task_struct *p, 661 struct cgroup *from, struct cgroup *to); 662 663 /** 664 * @cgroup_set_weight: A cgroup's weight is being changed 665 * @cgrp: cgroup whose weight is being updated 666 * @weight: new weight [1..10000] 667 * 668 * Update @cgrp's weight to @weight. 669 */ 670 void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); 671 672 /** 673 * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed 674 * @cgrp: cgroup whose bandwidth is being updated 675 * @period_us: bandwidth control period 676 * @quota_us: bandwidth control quota 677 * @burst_us: bandwidth control burst 678 * 679 * Update @cgrp's bandwidth control parameters. This is from the cpu.max 680 * cgroup interface. 681 * 682 * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled 683 * to. For example, if @period_us is 1_000_000 and @quota_us is 684 * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be 685 * interpreted in the same fashion and specifies how much @cgrp can 686 * burst temporarily. The specific control mechanism and thus the 687 * interpretation of @period_us and burstiness is upto to the BPF 688 * scheduler. 689 */ 690 void (*cgroup_set_bandwidth)(struct cgroup *cgrp, 691 u64 period_us, u64 quota_us, u64 burst_us); 692 693 #endif /* CONFIG_EXT_GROUP_SCHED */ 694 695 /* 696 * All online ops must come before ops.cpu_online(). 697 */ 698 699 /** 700 * @cpu_online: A CPU became online 701 * @cpu: CPU which just came up 702 * 703 * @cpu just came online. @cpu will not call ops.enqueue() or 704 * ops.dispatch(), nor run tasks associated with other CPUs beforehand. 705 */ 706 void (*cpu_online)(s32 cpu); 707 708 /** 709 * @cpu_offline: A CPU is going offline 710 * @cpu: CPU which is going offline 711 * 712 * @cpu is going offline. @cpu will not call ops.enqueue() or 713 * ops.dispatch(), nor run tasks associated with other CPUs afterwards. 714 */ 715 void (*cpu_offline)(s32 cpu); 716 717 /* 718 * All CPU hotplug ops must come before ops.init(). 719 */ 720 721 /** 722 * @init: Initialize the BPF scheduler 723 */ 724 s32 (*init)(void); 725 726 /** 727 * @exit: Clean up after the BPF scheduler 728 * @info: Exit info 729 * 730 * ops.exit() is also called on ops.init() failure, which is a bit 731 * unusual. This is to allow rich reporting through @info on how 732 * ops.init() failed. 733 */ 734 void (*exit)(struct scx_exit_info *info); 735 736 /** 737 * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch 738 */ 739 u32 dispatch_max_batch; 740 741 /** 742 * @flags: %SCX_OPS_* flags 743 */ 744 u64 flags; 745 746 /** 747 * @timeout_ms: The maximum amount of time, in milliseconds, that a 748 * runnable task should be able to wait before being scheduled. The 749 * maximum timeout may not exceed the default timeout of 30 seconds. 750 * 751 * Defaults to the maximum allowed timeout value of 30 seconds. 752 */ 753 u32 timeout_ms; 754 755 /** 756 * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default 757 * value of 32768 is used. 758 */ 759 u32 exit_dump_len; 760 761 /** 762 * @hotplug_seq: A sequence number that may be set by the scheduler to 763 * detect when a hotplug event has occurred during the loading process. 764 * If 0, no detection occurs. Otherwise, the scheduler will fail to 765 * load if the sequence number does not match @scx_hotplug_seq on the 766 * enable path. 767 */ 768 u64 hotplug_seq; 769 770 /** 771 * @name: BPF scheduler's name 772 * 773 * Must be a non-zero valid BPF object name including only isalnum(), 774 * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the 775 * BPF scheduler is enabled. 776 */ 777 char name[SCX_OPS_NAME_LEN]; 778 779 /* internal use only, must be NULL */ 780 void *priv; 781 }; 782 783 enum scx_opi { 784 SCX_OPI_BEGIN = 0, 785 SCX_OPI_NORMAL_BEGIN = 0, 786 SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), 787 SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), 788 SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), 789 SCX_OPI_END = SCX_OP_IDX(init), 790 }; 791 792 /* 793 * Collection of event counters. Event types are placed in descending order. 794 */ 795 struct scx_event_stats { 796 /* 797 * If ops.select_cpu() returns a CPU which can't be used by the task, 798 * the core scheduler code silently picks a fallback CPU. 799 */ 800 s64 SCX_EV_SELECT_CPU_FALLBACK; 801 802 /* 803 * When dispatching to a local DSQ, the CPU may have gone offline in 804 * the meantime. In this case, the task is bounced to the global DSQ. 805 */ 806 s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; 807 808 /* 809 * If SCX_OPS_ENQ_LAST is not set, the number of times that a task 810 * continued to run because there were no other tasks on the CPU. 811 */ 812 s64 SCX_EV_DISPATCH_KEEP_LAST; 813 814 /* 815 * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task 816 * is dispatched to a local DSQ when exiting. 817 */ 818 s64 SCX_EV_ENQ_SKIP_EXITING; 819 820 /* 821 * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a 822 * migration disabled task skips ops.enqueue() and is dispatched to its 823 * local DSQ. 824 */ 825 s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; 826 827 /* 828 * Total number of times a task's time slice was refilled with the 829 * default value (SCX_SLICE_DFL). 830 */ 831 s64 SCX_EV_REFILL_SLICE_DFL; 832 833 /* 834 * The total duration of bypass modes in nanoseconds. 835 */ 836 s64 SCX_EV_BYPASS_DURATION; 837 838 /* 839 * The number of tasks dispatched in the bypassing mode. 840 */ 841 s64 SCX_EV_BYPASS_DISPATCH; 842 843 /* 844 * The number of times the bypassing mode has been activated. 845 */ 846 s64 SCX_EV_BYPASS_ACTIVATE; 847 }; 848 849 struct scx_sched_pcpu { 850 /* 851 * The event counters are in a per-CPU variable to minimize the 852 * accounting overhead. A system-wide view on the event counter is 853 * constructed when requested by scx_bpf_events(). 854 */ 855 struct scx_event_stats event_stats; 856 }; 857 858 struct scx_sched { 859 struct sched_ext_ops ops; 860 DECLARE_BITMAP(has_op, SCX_OPI_END); 861 862 /* 863 * Dispatch queues. 864 * 865 * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. 866 * This is to avoid live-locking in bypass mode where all tasks are 867 * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If 868 * per-node split isn't sufficient, it can be further split. 869 */ 870 struct rhashtable dsq_hash; 871 struct scx_dispatch_q **global_dsqs; 872 struct scx_sched_pcpu __percpu *pcpu; 873 874 bool warned_zero_slice:1; 875 bool warned_deprecated_rq:1; 876 877 atomic_t exit_kind; 878 struct scx_exit_info *exit_info; 879 880 struct kobject kobj; 881 882 struct kthread_worker *helper; 883 struct irq_work error_irq_work; 884 struct kthread_work disable_work; 885 struct rcu_work rcu_work; 886 }; 887 888 enum scx_wake_flags { 889 /* expose select WF_* flags as enums */ 890 SCX_WAKE_FORK = WF_FORK, 891 SCX_WAKE_TTWU = WF_TTWU, 892 SCX_WAKE_SYNC = WF_SYNC, 893 }; 894 895 enum scx_enq_flags { 896 /* expose select ENQUEUE_* flags as enums */ 897 SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, 898 SCX_ENQ_HEAD = ENQUEUE_HEAD, 899 SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, 900 901 /* high 32bits are SCX specific */ 902 903 /* 904 * Set the following to trigger preemption when calling 905 * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the 906 * current task is cleared to zero and the CPU is kicked into the 907 * scheduling path. Implies %SCX_ENQ_HEAD. 908 */ 909 SCX_ENQ_PREEMPT = 1LLU << 32, 910 911 /* 912 * The task being enqueued was previously enqueued on the current CPU's 913 * %SCX_DSQ_LOCAL, but was removed from it in a call to the 914 * scx_bpf_reenqueue_local() kfunc. If scx_bpf_reenqueue_local() was 915 * invoked in a ->cpu_release() callback, and the task is again 916 * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the 917 * task will not be scheduled on the CPU until at least the next invocation 918 * of the ->cpu_acquire() callback. 919 */ 920 SCX_ENQ_REENQ = 1LLU << 40, 921 922 /* 923 * The task being enqueued is the only task available for the cpu. By 924 * default, ext core keeps executing such tasks but when 925 * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the 926 * %SCX_ENQ_LAST flag set. 927 * 928 * The BPF scheduler is responsible for triggering a follow-up 929 * scheduling event. Otherwise, Execution may stall. 930 */ 931 SCX_ENQ_LAST = 1LLU << 41, 932 933 /* high 8 bits are internal */ 934 __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, 935 936 SCX_ENQ_CLEAR_OPSS = 1LLU << 56, 937 SCX_ENQ_DSQ_PRIQ = 1LLU << 57, 938 }; 939 940 enum scx_deq_flags { 941 /* expose select DEQUEUE_* flags as enums */ 942 SCX_DEQ_SLEEP = DEQUEUE_SLEEP, 943 944 /* high 32bits are SCX specific */ 945 946 /* 947 * The generic core-sched layer decided to execute the task even though 948 * it hasn't been dispatched yet. Dequeue from the BPF side. 949 */ 950 SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, 951 }; 952 953 enum scx_pick_idle_cpu_flags { 954 SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ 955 SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ 956 }; 957 958 enum scx_kick_flags { 959 /* 960 * Kick the target CPU if idle. Guarantees that the target CPU goes 961 * through at least one full scheduling cycle before going idle. If the 962 * target CPU can be determined to be currently not idle and going to go 963 * through a scheduling cycle before going idle, noop. 964 */ 965 SCX_KICK_IDLE = 1LLU << 0, 966 967 /* 968 * Preempt the current task and execute the dispatch path. If the 969 * current task of the target CPU is an SCX task, its ->scx.slice is 970 * cleared to zero before the scheduling path is invoked so that the 971 * task expires and the dispatch path is invoked. 972 */ 973 SCX_KICK_PREEMPT = 1LLU << 1, 974 975 /* 976 * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will 977 * return after the target CPU finishes picking the next task. 978 */ 979 SCX_KICK_WAIT = 1LLU << 2, 980 }; 981 982 enum scx_tg_flags { 983 SCX_TG_ONLINE = 1U << 0, 984 SCX_TG_INITED = 1U << 1, 985 }; 986 987 enum scx_enable_state { 988 SCX_ENABLING, 989 SCX_ENABLED, 990 SCX_DISABLING, 991 SCX_DISABLED, 992 }; 993 994 static const char *scx_enable_state_str[] = { 995 [SCX_ENABLING] = "enabling", 996 [SCX_ENABLED] = "enabled", 997 [SCX_DISABLING] = "disabling", 998 [SCX_DISABLED] = "disabled", 999 }; 1000 1001 /* 1002 * sched_ext_entity->ops_state 1003 * 1004 * Used to track the task ownership between the SCX core and the BPF scheduler. 1005 * State transitions look as follows: 1006 * 1007 * NONE -> QUEUEING -> QUEUED -> DISPATCHING 1008 * ^ | | 1009 * | v v 1010 * \-------------------------------/ 1011 * 1012 * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call 1013 * sites for explanations on the conditions being waited upon and why they are 1014 * safe. Transitions out of them into NONE or QUEUED must store_release and the 1015 * waiters should load_acquire. 1016 * 1017 * Tracking scx_ops_state enables sched_ext core to reliably determine whether 1018 * any given task can be dispatched by the BPF scheduler at all times and thus 1019 * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler 1020 * to try to dispatch any task anytime regardless of its state as the SCX core 1021 * can safely reject invalid dispatches. 1022 */ 1023 enum scx_ops_state { 1024 SCX_OPSS_NONE, /* owned by the SCX core */ 1025 SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ 1026 SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ 1027 SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ 1028 1029 /* 1030 * QSEQ brands each QUEUED instance so that, when dispatch races 1031 * dequeue/requeue, the dispatcher can tell whether it still has a claim 1032 * on the task being dispatched. 1033 * 1034 * As some 32bit archs can't do 64bit store_release/load_acquire, 1035 * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on 1036 * 32bit machines. The dispatch race window QSEQ protects is very narrow 1037 * and runs with IRQ disabled. 30 bits should be sufficient. 1038 */ 1039 SCX_OPSS_QSEQ_SHIFT = 2, 1040 }; 1041 1042 /* Use macros to ensure that the type is unsigned long for the masks */ 1043 #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) 1044 #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) 1045 1046 DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); 1047 1048 /* 1049 * Return the rq currently locked from an scx callback, or NULL if no rq is 1050 * locked. 1051 */ 1052 static inline struct rq *scx_locked_rq(void) 1053 { 1054 return __this_cpu_read(scx_locked_rq_state); 1055 } 1056 1057 static inline bool scx_kf_allowed_if_unlocked(void) 1058 { 1059 return !current->scx.kf_mask; 1060 } 1061 1062 static inline bool scx_rq_bypassing(struct rq *rq) 1063 { 1064 return unlikely(rq->scx.flags & SCX_RQ_BYPASSING); 1065 } 1066