1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. 6 * Copyright (c) 2025 Tejun Heo <tj@kernel.org> 7 */ 8 #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) 9 10 enum scx_consts { 11 SCX_DSP_DFL_MAX_BATCH = 32, 12 SCX_DSP_MAX_LOOPS = 32, 13 SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, 14 15 SCX_EXIT_BT_LEN = 64, 16 SCX_EXIT_MSG_LEN = 1024, 17 SCX_EXIT_DUMP_DFL_LEN = 32768, 18 19 SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, 20 21 /* 22 * Iterating all tasks may take a while. Periodically drop 23 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. 24 */ 25 SCX_TASK_ITER_BATCH = 32, 26 }; 27 28 enum scx_exit_kind { 29 SCX_EXIT_NONE, 30 SCX_EXIT_DONE, 31 32 SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ 33 SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ 34 SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ 35 SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ 36 37 SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ 38 SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ 39 SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ 40 }; 41 42 /* 43 * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), 44 * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes 45 * are 64bit of the format: 46 * 47 * Bits: [63 .. 48 47 .. 32 31 .. 0] 48 * [ SYS ACT ] [ SYS RSN ] [ USR ] 49 * 50 * SYS ACT: System-defined exit actions 51 * SYS RSN: System-defined exit reasons 52 * USR : User-defined exit codes and reasons 53 * 54 * Using the above, users may communicate intention and context by ORing system 55 * actions and/or system reasons with a user-defined exit code. 56 */ 57 enum scx_exit_code { 58 /* Reasons */ 59 SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, 60 61 /* Actions */ 62 SCX_ECODE_ACT_RESTART = 1LLU << 48, 63 }; 64 65 enum scx_exit_flags { 66 /* 67 * ops.exit() may be called even if the loading failed before ops.init() 68 * finishes successfully. This is because ops.exit() allows rich exit 69 * info communication. The following flag indicates whether ops.init() 70 * finished successfully. 71 */ 72 SCX_EFLAG_INITIALIZED, 73 }; 74 75 /* 76 * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is 77 * being disabled. 78 */ 79 struct scx_exit_info { 80 /* %SCX_EXIT_* - broad category of the exit reason */ 81 enum scx_exit_kind kind; 82 83 /* exit code if gracefully exiting */ 84 s64 exit_code; 85 86 /* %SCX_EFLAG_* */ 87 u64 flags; 88 89 /* textual representation of the above */ 90 const char *reason; 91 92 /* backtrace if exiting due to an error */ 93 unsigned long *bt; 94 u32 bt_len; 95 96 /* informational message */ 97 char *msg; 98 99 /* debug dump */ 100 char *dump; 101 }; 102 103 /* sched_ext_ops.flags */ 104 enum scx_ops_flags { 105 /* 106 * Keep built-in idle tracking even if ops.update_idle() is implemented. 107 */ 108 SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, 109 110 /* 111 * By default, if there are no other task to run on the CPU, ext core 112 * keeps running the current task even after its slice expires. If this 113 * flag is specified, such tasks are passed to ops.enqueue() with 114 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. 115 */ 116 SCX_OPS_ENQ_LAST = 1LLU << 1, 117 118 /* 119 * An exiting task may schedule after PF_EXITING is set. In such cases, 120 * bpf_task_from_pid() may not be able to find the task and if the BPF 121 * scheduler depends on pid lookup for dispatching, the task will be 122 * lost leading to various issues including RCU grace period stalls. 123 * 124 * To mask this problem, by default, unhashed tasks are automatically 125 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't 126 * depend on pid lookups and wants to handle these tasks directly, the 127 * following flag can be used. 128 */ 129 SCX_OPS_ENQ_EXITING = 1LLU << 2, 130 131 /* 132 * If set, only tasks with policy set to SCHED_EXT are attached to 133 * sched_ext. If clear, SCHED_NORMAL tasks are also included. 134 */ 135 SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, 136 137 /* 138 * A migration disabled task can only execute on its current CPU. By 139 * default, such tasks are automatically put on the CPU's local DSQ with 140 * the default slice on enqueue. If this ops flag is set, they also go 141 * through ops.enqueue(). 142 * 143 * A migration disabled task never invokes ops.select_cpu() as it can 144 * only select the current CPU. Also, p->cpus_ptr will only contain its 145 * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr 146 * and thus may disagree with cpumask_weight(p->cpus_ptr). 147 */ 148 SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4, 149 150 /* 151 * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes 152 * ops.enqueue() on the ops.select_cpu() selected or the wakee's 153 * previous CPU via IPI (inter-processor interrupt) to reduce cacheline 154 * transfers. When this optimization is enabled, ops.select_cpu() is 155 * skipped in some cases (when racing against the wakee switching out). 156 * As the BPF scheduler may depend on ops.select_cpu() being invoked 157 * during wakeups, queued wakeup is disabled by default. 158 * 159 * If this ops flag is set, queued wakeup optimization is enabled and 160 * the BPF scheduler must be able to handle ops.enqueue() invoked on the 161 * wakee's CPU without preceding ops.select_cpu() even for tasks which 162 * may be executed on multiple CPUs. 163 */ 164 SCX_OPS_ALLOW_QUEUED_WAKEUP = 1LLU << 5, 165 166 /* 167 * If set, enable per-node idle cpumasks. If clear, use a single global 168 * flat idle cpumask. 169 */ 170 SCX_OPS_BUILTIN_IDLE_PER_NODE = 1LLU << 6, 171 172 /* 173 * CPU cgroup support flags 174 */ 175 SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16, /* DEPRECATED, will be removed on 6.18 */ 176 177 SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | 178 SCX_OPS_ENQ_LAST | 179 SCX_OPS_ENQ_EXITING | 180 SCX_OPS_ENQ_MIGRATION_DISABLED | 181 SCX_OPS_ALLOW_QUEUED_WAKEUP | 182 SCX_OPS_SWITCH_PARTIAL | 183 SCX_OPS_BUILTIN_IDLE_PER_NODE | 184 SCX_OPS_HAS_CGROUP_WEIGHT, 185 186 /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ 187 __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, 188 189 SCX_OPS_HAS_CPU_PREEMPT = 1LLU << 56, 190 }; 191 192 /* argument container for ops.init_task() */ 193 struct scx_init_task_args { 194 /* 195 * Set if ops.init_task() is being invoked on the fork path, as opposed 196 * to the scheduler transition path. 197 */ 198 bool fork; 199 #ifdef CONFIG_EXT_GROUP_SCHED 200 /* the cgroup the task is joining */ 201 struct cgroup *cgroup; 202 #endif 203 }; 204 205 /* argument container for ops.exit_task() */ 206 struct scx_exit_task_args { 207 /* Whether the task exited before running on sched_ext. */ 208 bool cancelled; 209 }; 210 211 /* argument container for ops->cgroup_init() */ 212 struct scx_cgroup_init_args { 213 /* the weight of the cgroup [1..10000] */ 214 u32 weight; 215 216 /* bandwidth control parameters from cpu.max and cpu.max.burst */ 217 u64 bw_period_us; 218 u64 bw_quota_us; 219 u64 bw_burst_us; 220 }; 221 222 enum scx_cpu_preempt_reason { 223 /* next task is being scheduled by &sched_class_rt */ 224 SCX_CPU_PREEMPT_RT, 225 /* next task is being scheduled by &sched_class_dl */ 226 SCX_CPU_PREEMPT_DL, 227 /* next task is being scheduled by &sched_class_stop */ 228 SCX_CPU_PREEMPT_STOP, 229 /* unknown reason for SCX being preempted */ 230 SCX_CPU_PREEMPT_UNKNOWN, 231 }; 232 233 /* 234 * Argument container for ops->cpu_acquire(). Currently empty, but may be 235 * expanded in the future. 236 */ 237 struct scx_cpu_acquire_args {}; 238 239 /* argument container for ops->cpu_release() */ 240 struct scx_cpu_release_args { 241 /* the reason the CPU was preempted */ 242 enum scx_cpu_preempt_reason reason; 243 244 /* the task that's going to be scheduled on the CPU */ 245 struct task_struct *task; 246 }; 247 248 /* 249 * Informational context provided to dump operations. 250 */ 251 struct scx_dump_ctx { 252 enum scx_exit_kind kind; 253 s64 exit_code; 254 const char *reason; 255 u64 at_ns; 256 u64 at_jiffies; 257 }; 258 259 /** 260 * struct sched_ext_ops - Operation table for BPF scheduler implementation 261 * 262 * A BPF scheduler can implement an arbitrary scheduling policy by 263 * implementing and loading operations in this table. Note that a userland 264 * scheduling policy can also be implemented using the BPF scheduler 265 * as a shim layer. 266 */ 267 struct sched_ext_ops { 268 /** 269 * @select_cpu: Pick the target CPU for a task which is being woken up 270 * @p: task being woken up 271 * @prev_cpu: the cpu @p was on before sleeping 272 * @wake_flags: SCX_WAKE_* 273 * 274 * Decision made here isn't final. @p may be moved to any CPU while it 275 * is getting dispatched for execution later. However, as @p is not on 276 * the rq at this point, getting the eventual execution CPU right here 277 * saves a small bit of overhead down the line. 278 * 279 * If an idle CPU is returned, the CPU is kicked and will try to 280 * dispatch. While an explicit custom mechanism can be added, 281 * select_cpu() serves as the default way to wake up idle CPUs. 282 * 283 * @p may be inserted into a DSQ directly by calling 284 * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped. 285 * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ 286 * of the CPU returned by this operation. 287 * 288 * Note that select_cpu() is never called for tasks that can only run 289 * on a single CPU or tasks with migration disabled, as they don't have 290 * the option to select a different CPU. See select_task_rq() for 291 * details. 292 */ 293 s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 294 295 /** 296 * @enqueue: Enqueue a task on the BPF scheduler 297 * @p: task being enqueued 298 * @enq_flags: %SCX_ENQ_* 299 * 300 * @p is ready to run. Insert directly into a DSQ by calling 301 * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly 302 * inserted, the bpf scheduler owns @p and if it fails to dispatch @p, 303 * the task will stall. 304 * 305 * If @p was inserted into a DSQ from ops.select_cpu(), this callback is 306 * skipped. 307 */ 308 void (*enqueue)(struct task_struct *p, u64 enq_flags); 309 310 /** 311 * @dequeue: Remove a task from the BPF scheduler 312 * @p: task being dequeued 313 * @deq_flags: %SCX_DEQ_* 314 * 315 * Remove @p from the BPF scheduler. This is usually called to isolate 316 * the task while updating its scheduling properties (e.g. priority). 317 * 318 * The ext core keeps track of whether the BPF side owns a given task or 319 * not and can gracefully ignore spurious dispatches from BPF side, 320 * which makes it safe to not implement this method. However, depending 321 * on the scheduling logic, this can lead to confusing behaviors - e.g. 322 * scheduling position not being updated across a priority change. 323 */ 324 void (*dequeue)(struct task_struct *p, u64 deq_flags); 325 326 /** 327 * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs 328 * @cpu: CPU to dispatch tasks for 329 * @prev: previous task being switched out 330 * 331 * Called when a CPU's local dsq is empty. The operation should dispatch 332 * one or more tasks from the BPF scheduler into the DSQs using 333 * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ 334 * using scx_bpf_dsq_move_to_local(). 335 * 336 * The maximum number of times scx_bpf_dsq_insert() can be called 337 * without an intervening scx_bpf_dsq_move_to_local() is specified by 338 * ops.dispatch_max_batch. See the comments on top of the two functions 339 * for more details. 340 * 341 * When not %NULL, @prev is an SCX task with its slice depleted. If 342 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in 343 * @prev->scx.flags, it is not enqueued yet and will be enqueued after 344 * ops.dispatch() returns. To keep executing @prev, return without 345 * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST. 346 */ 347 void (*dispatch)(s32 cpu, struct task_struct *prev); 348 349 /** 350 * @tick: Periodic tick 351 * @p: task running currently 352 * 353 * This operation is called every 1/HZ seconds on CPUs which are 354 * executing an SCX task. Setting @p->scx.slice to 0 will trigger an 355 * immediate dispatch cycle on the CPU. 356 */ 357 void (*tick)(struct task_struct *p); 358 359 /** 360 * @runnable: A task is becoming runnable on its associated CPU 361 * @p: task becoming runnable 362 * @enq_flags: %SCX_ENQ_* 363 * 364 * This and the following three functions can be used to track a task's 365 * execution state transitions. A task becomes ->runnable() on a CPU, 366 * and then goes through one or more ->running() and ->stopping() pairs 367 * as it runs on the CPU, and eventually becomes ->quiescent() when it's 368 * done running on the CPU. 369 * 370 * @p is becoming runnable on the CPU because it's 371 * 372 * - waking up (%SCX_ENQ_WAKEUP) 373 * - being moved from another CPU 374 * - being restored after temporarily taken off the queue for an 375 * attribute change. 376 * 377 * This and ->enqueue() are related but not coupled. This operation 378 * notifies @p's state transition and may not be followed by ->enqueue() 379 * e.g. when @p is being dispatched to a remote CPU, or when @p is 380 * being enqueued on a CPU experiencing a hotplug event. Likewise, a 381 * task may be ->enqueue()'d without being preceded by this operation 382 * e.g. after exhausting its slice. 383 */ 384 void (*runnable)(struct task_struct *p, u64 enq_flags); 385 386 /** 387 * @running: A task is starting to run on its associated CPU 388 * @p: task starting to run 389 * 390 * Note that this callback may be called from a CPU other than the 391 * one the task is going to run on. This can happen when a task 392 * property is changed (i.e., affinity), since scx_next_task_scx(), 393 * which triggers this callback, may run on a CPU different from 394 * the task's assigned CPU. 395 * 396 * Therefore, always use scx_bpf_task_cpu(@p) to determine the 397 * target CPU the task is going to use. 398 * 399 * See ->runnable() for explanation on the task state notifiers. 400 */ 401 void (*running)(struct task_struct *p); 402 403 /** 404 * @stopping: A task is stopping execution 405 * @p: task stopping to run 406 * @runnable: is task @p still runnable? 407 * 408 * Note that this callback may be called from a CPU other than the 409 * one the task was running on. This can happen when a task 410 * property is changed (i.e., affinity), since dequeue_task_scx(), 411 * which triggers this callback, may run on a CPU different from 412 * the task's assigned CPU. 413 * 414 * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU 415 * the task was running on. 416 * 417 * See ->runnable() for explanation on the task state notifiers. If 418 * !@runnable, ->quiescent() will be invoked after this operation 419 * returns. 420 */ 421 void (*stopping)(struct task_struct *p, bool runnable); 422 423 /** 424 * @quiescent: A task is becoming not runnable on its associated CPU 425 * @p: task becoming not runnable 426 * @deq_flags: %SCX_DEQ_* 427 * 428 * See ->runnable() for explanation on the task state notifiers. 429 * 430 * @p is becoming quiescent on the CPU because it's 431 * 432 * - sleeping (%SCX_DEQ_SLEEP) 433 * - being moved to another CPU 434 * - being temporarily taken off the queue for an attribute change 435 * (%SCX_DEQ_SAVE) 436 * 437 * This and ->dequeue() are related but not coupled. This operation 438 * notifies @p's state transition and may not be preceded by ->dequeue() 439 * e.g. when @p is being dispatched to a remote CPU. 440 */ 441 void (*quiescent)(struct task_struct *p, u64 deq_flags); 442 443 /** 444 * @yield: Yield CPU 445 * @from: yielding task 446 * @to: optional yield target task 447 * 448 * If @to is NULL, @from is yielding the CPU to other runnable tasks. 449 * The BPF scheduler should ensure that other available tasks are 450 * dispatched before the yielding task. Return value is ignored in this 451 * case. 452 * 453 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf 454 * scheduler can implement the request, return %true; otherwise, %false. 455 */ 456 bool (*yield)(struct task_struct *from, struct task_struct *to); 457 458 /** 459 * @core_sched_before: Task ordering for core-sched 460 * @a: task A 461 * @b: task B 462 * 463 * Used by core-sched to determine the ordering between two tasks. See 464 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on 465 * core-sched. 466 * 467 * Both @a and @b are runnable and may or may not currently be queued on 468 * the BPF scheduler. Should return %true if @a should run before @b. 469 * %false if there's no required ordering or @b should run before @a. 470 * 471 * If not specified, the default is ordering them according to when they 472 * became runnable. 473 */ 474 bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); 475 476 /** 477 * @set_weight: Set task weight 478 * @p: task to set weight for 479 * @weight: new weight [1..10000] 480 * 481 * Update @p's weight to @weight. 482 */ 483 void (*set_weight)(struct task_struct *p, u32 weight); 484 485 /** 486 * @set_cpumask: Set CPU affinity 487 * @p: task to set CPU affinity for 488 * @cpumask: cpumask of cpus that @p can run on 489 * 490 * Update @p's CPU affinity to @cpumask. 491 */ 492 void (*set_cpumask)(struct task_struct *p, 493 const struct cpumask *cpumask); 494 495 /** 496 * @update_idle: Update the idle state of a CPU 497 * @cpu: CPU to update the idle state for 498 * @idle: whether entering or exiting the idle state 499 * 500 * This operation is called when @rq's CPU goes or leaves the idle 501 * state. By default, implementing this operation disables the built-in 502 * idle CPU tracking and the following helpers become unavailable: 503 * 504 * - scx_bpf_select_cpu_dfl() 505 * - scx_bpf_select_cpu_and() 506 * - scx_bpf_test_and_clear_cpu_idle() 507 * - scx_bpf_pick_idle_cpu() 508 * 509 * The user also must implement ops.select_cpu() as the default 510 * implementation relies on scx_bpf_select_cpu_dfl(). 511 * 512 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle 513 * tracking. 514 */ 515 void (*update_idle)(s32 cpu, bool idle); 516 517 /** 518 * @cpu_acquire: A CPU is becoming available to the BPF scheduler 519 * @cpu: The CPU being acquired by the BPF scheduler. 520 * @args: Acquire arguments, see the struct definition. 521 * 522 * A CPU that was previously released from the BPF scheduler is now once 523 * again under its control. 524 */ 525 void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); 526 527 /** 528 * @cpu_release: A CPU is taken away from the BPF scheduler 529 * @cpu: The CPU being released by the BPF scheduler. 530 * @args: Release arguments, see the struct definition. 531 * 532 * The specified CPU is no longer under the control of the BPF 533 * scheduler. This could be because it was preempted by a higher 534 * priority sched_class, though there may be other reasons as well. The 535 * caller should consult @args->reason to determine the cause. 536 */ 537 void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); 538 539 /** 540 * @init_task: Initialize a task to run in a BPF scheduler 541 * @p: task to initialize for BPF scheduling 542 * @args: init arguments, see the struct definition 543 * 544 * Either we're loading a BPF scheduler or a new task is being forked. 545 * Initialize @p for BPF scheduling. This operation may block and can 546 * be used for allocations, and is called exactly once for a task. 547 * 548 * Return 0 for success, -errno for failure. An error return while 549 * loading will abort loading of the BPF scheduler. During a fork, it 550 * will abort that specific fork. 551 */ 552 s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); 553 554 /** 555 * @exit_task: Exit a previously-running task from the system 556 * @p: task to exit 557 * @args: exit arguments, see the struct definition 558 * 559 * @p is exiting or the BPF scheduler is being unloaded. Perform any 560 * necessary cleanup for @p. 561 */ 562 void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); 563 564 /** 565 * @enable: Enable BPF scheduling for a task 566 * @p: task to enable BPF scheduling for 567 * 568 * Enable @p for BPF scheduling. enable() is called on @p any time it 569 * enters SCX, and is always paired with a matching disable(). 570 */ 571 void (*enable)(struct task_struct *p); 572 573 /** 574 * @disable: Disable BPF scheduling for a task 575 * @p: task to disable BPF scheduling for 576 * 577 * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. 578 * Disable BPF scheduling for @p. A disable() call is always matched 579 * with a prior enable() call. 580 */ 581 void (*disable)(struct task_struct *p); 582 583 /** 584 * @dump: Dump BPF scheduler state on error 585 * @ctx: debug dump context 586 * 587 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. 588 */ 589 void (*dump)(struct scx_dump_ctx *ctx); 590 591 /** 592 * @dump_cpu: Dump BPF scheduler state for a CPU on error 593 * @ctx: debug dump context 594 * @cpu: CPU to generate debug dump for 595 * @idle: @cpu is currently idle without any runnable tasks 596 * 597 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 598 * @cpu. If @idle is %true and this operation doesn't produce any 599 * output, @cpu is skipped for dump. 600 */ 601 void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); 602 603 /** 604 * @dump_task: Dump BPF scheduler state for a runnable task on error 605 * @ctx: debug dump context 606 * @p: runnable task to generate debug dump for 607 * 608 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 609 * @p. 610 */ 611 void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); 612 613 #ifdef CONFIG_EXT_GROUP_SCHED 614 /** 615 * @cgroup_init: Initialize a cgroup 616 * @cgrp: cgroup being initialized 617 * @args: init arguments, see the struct definition 618 * 619 * Either the BPF scheduler is being loaded or @cgrp created, initialize 620 * @cgrp for sched_ext. This operation may block. 621 * 622 * Return 0 for success, -errno for failure. An error return while 623 * loading will abort loading of the BPF scheduler. During cgroup 624 * creation, it will abort the specific cgroup creation. 625 */ 626 s32 (*cgroup_init)(struct cgroup *cgrp, 627 struct scx_cgroup_init_args *args); 628 629 /** 630 * @cgroup_exit: Exit a cgroup 631 * @cgrp: cgroup being exited 632 * 633 * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit 634 * @cgrp for sched_ext. This operation my block. 635 */ 636 void (*cgroup_exit)(struct cgroup *cgrp); 637 638 /** 639 * @cgroup_prep_move: Prepare a task to be moved to a different cgroup 640 * @p: task being moved 641 * @from: cgroup @p is being moved from 642 * @to: cgroup @p is being moved to 643 * 644 * Prepare @p for move from cgroup @from to @to. This operation may 645 * block and can be used for allocations. 646 * 647 * Return 0 for success, -errno for failure. An error return aborts the 648 * migration. 649 */ 650 s32 (*cgroup_prep_move)(struct task_struct *p, 651 struct cgroup *from, struct cgroup *to); 652 653 /** 654 * @cgroup_move: Commit cgroup move 655 * @p: task being moved 656 * @from: cgroup @p is being moved from 657 * @to: cgroup @p is being moved to 658 * 659 * Commit the move. @p is dequeued during this operation. 660 */ 661 void (*cgroup_move)(struct task_struct *p, 662 struct cgroup *from, struct cgroup *to); 663 664 /** 665 * @cgroup_cancel_move: Cancel cgroup move 666 * @p: task whose cgroup move is being canceled 667 * @from: cgroup @p was being moved from 668 * @to: cgroup @p was being moved to 669 * 670 * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). 671 * Undo the preparation. 672 */ 673 void (*cgroup_cancel_move)(struct task_struct *p, 674 struct cgroup *from, struct cgroup *to); 675 676 /** 677 * @cgroup_set_weight: A cgroup's weight is being changed 678 * @cgrp: cgroup whose weight is being updated 679 * @weight: new weight [1..10000] 680 * 681 * Update @cgrp's weight to @weight. 682 */ 683 void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); 684 685 /** 686 * @cgroup_set_bandwidth: A cgroup's bandwidth is being changed 687 * @cgrp: cgroup whose bandwidth is being updated 688 * @period_us: bandwidth control period 689 * @quota_us: bandwidth control quota 690 * @burst_us: bandwidth control burst 691 * 692 * Update @cgrp's bandwidth control parameters. This is from the cpu.max 693 * cgroup interface. 694 * 695 * @quota_us / @period_us determines the CPU bandwidth @cgrp is entitled 696 * to. For example, if @period_us is 1_000_000 and @quota_us is 697 * 2_500_000. @cgrp is entitled to 2.5 CPUs. @burst_us can be 698 * interpreted in the same fashion and specifies how much @cgrp can 699 * burst temporarily. The specific control mechanism and thus the 700 * interpretation of @period_us and burstiness is upto to the BPF 701 * scheduler. 702 */ 703 void (*cgroup_set_bandwidth)(struct cgroup *cgrp, 704 u64 period_us, u64 quota_us, u64 burst_us); 705 706 #endif /* CONFIG_EXT_GROUP_SCHED */ 707 708 /* 709 * All online ops must come before ops.cpu_online(). 710 */ 711 712 /** 713 * @cpu_online: A CPU became online 714 * @cpu: CPU which just came up 715 * 716 * @cpu just came online. @cpu will not call ops.enqueue() or 717 * ops.dispatch(), nor run tasks associated with other CPUs beforehand. 718 */ 719 void (*cpu_online)(s32 cpu); 720 721 /** 722 * @cpu_offline: A CPU is going offline 723 * @cpu: CPU which is going offline 724 * 725 * @cpu is going offline. @cpu will not call ops.enqueue() or 726 * ops.dispatch(), nor run tasks associated with other CPUs afterwards. 727 */ 728 void (*cpu_offline)(s32 cpu); 729 730 /* 731 * All CPU hotplug ops must come before ops.init(). 732 */ 733 734 /** 735 * @init: Initialize the BPF scheduler 736 */ 737 s32 (*init)(void); 738 739 /** 740 * @exit: Clean up after the BPF scheduler 741 * @info: Exit info 742 * 743 * ops.exit() is also called on ops.init() failure, which is a bit 744 * unusual. This is to allow rich reporting through @info on how 745 * ops.init() failed. 746 */ 747 void (*exit)(struct scx_exit_info *info); 748 749 /** 750 * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch 751 */ 752 u32 dispatch_max_batch; 753 754 /** 755 * @flags: %SCX_OPS_* flags 756 */ 757 u64 flags; 758 759 /** 760 * @timeout_ms: The maximum amount of time, in milliseconds, that a 761 * runnable task should be able to wait before being scheduled. The 762 * maximum timeout may not exceed the default timeout of 30 seconds. 763 * 764 * Defaults to the maximum allowed timeout value of 30 seconds. 765 */ 766 u32 timeout_ms; 767 768 /** 769 * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default 770 * value of 32768 is used. 771 */ 772 u32 exit_dump_len; 773 774 /** 775 * @hotplug_seq: A sequence number that may be set by the scheduler to 776 * detect when a hotplug event has occurred during the loading process. 777 * If 0, no detection occurs. Otherwise, the scheduler will fail to 778 * load if the sequence number does not match @scx_hotplug_seq on the 779 * enable path. 780 */ 781 u64 hotplug_seq; 782 783 /** 784 * @name: BPF scheduler's name 785 * 786 * Must be a non-zero valid BPF object name including only isalnum(), 787 * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the 788 * BPF scheduler is enabled. 789 */ 790 char name[SCX_OPS_NAME_LEN]; 791 792 /* internal use only, must be NULL */ 793 void *priv; 794 }; 795 796 enum scx_opi { 797 SCX_OPI_BEGIN = 0, 798 SCX_OPI_NORMAL_BEGIN = 0, 799 SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), 800 SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), 801 SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), 802 SCX_OPI_END = SCX_OP_IDX(init), 803 }; 804 805 /* 806 * Collection of event counters. Event types are placed in descending order. 807 */ 808 struct scx_event_stats { 809 /* 810 * If ops.select_cpu() returns a CPU which can't be used by the task, 811 * the core scheduler code silently picks a fallback CPU. 812 */ 813 s64 SCX_EV_SELECT_CPU_FALLBACK; 814 815 /* 816 * When dispatching to a local DSQ, the CPU may have gone offline in 817 * the meantime. In this case, the task is bounced to the global DSQ. 818 */ 819 s64 SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE; 820 821 /* 822 * If SCX_OPS_ENQ_LAST is not set, the number of times that a task 823 * continued to run because there were no other tasks on the CPU. 824 */ 825 s64 SCX_EV_DISPATCH_KEEP_LAST; 826 827 /* 828 * If SCX_OPS_ENQ_EXITING is not set, the number of times that a task 829 * is dispatched to a local DSQ when exiting. 830 */ 831 s64 SCX_EV_ENQ_SKIP_EXITING; 832 833 /* 834 * If SCX_OPS_ENQ_MIGRATION_DISABLED is not set, the number of times a 835 * migration disabled task skips ops.enqueue() and is dispatched to its 836 * local DSQ. 837 */ 838 s64 SCX_EV_ENQ_SKIP_MIGRATION_DISABLED; 839 840 /* 841 * Total number of times a task's time slice was refilled with the 842 * default value (SCX_SLICE_DFL). 843 */ 844 s64 SCX_EV_REFILL_SLICE_DFL; 845 846 /* 847 * The total duration of bypass modes in nanoseconds. 848 */ 849 s64 SCX_EV_BYPASS_DURATION; 850 851 /* 852 * The number of tasks dispatched in the bypassing mode. 853 */ 854 s64 SCX_EV_BYPASS_DISPATCH; 855 856 /* 857 * The number of times the bypassing mode has been activated. 858 */ 859 s64 SCX_EV_BYPASS_ACTIVATE; 860 }; 861 862 struct scx_sched_pcpu { 863 /* 864 * The event counters are in a per-CPU variable to minimize the 865 * accounting overhead. A system-wide view on the event counter is 866 * constructed when requested by scx_bpf_events(). 867 */ 868 struct scx_event_stats event_stats; 869 }; 870 871 struct scx_sched { 872 struct sched_ext_ops ops; 873 DECLARE_BITMAP(has_op, SCX_OPI_END); 874 875 /* 876 * Dispatch queues. 877 * 878 * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. 879 * This is to avoid live-locking in bypass mode where all tasks are 880 * dispatched to %SCX_DSQ_GLOBAL and all CPUs consume from it. If 881 * per-node split isn't sufficient, it can be further split. 882 */ 883 struct rhashtable dsq_hash; 884 struct scx_dispatch_q **global_dsqs; 885 struct scx_sched_pcpu __percpu *pcpu; 886 887 bool warned_zero_slice:1; 888 bool warned_deprecated_rq:1; 889 890 atomic_t exit_kind; 891 struct scx_exit_info *exit_info; 892 893 struct kobject kobj; 894 895 struct kthread_worker *helper; 896 struct irq_work error_irq_work; 897 struct kthread_work disable_work; 898 struct rcu_work rcu_work; 899 }; 900 901 enum scx_wake_flags { 902 /* expose select WF_* flags as enums */ 903 SCX_WAKE_FORK = WF_FORK, 904 SCX_WAKE_TTWU = WF_TTWU, 905 SCX_WAKE_SYNC = WF_SYNC, 906 }; 907 908 enum scx_enq_flags { 909 /* expose select ENQUEUE_* flags as enums */ 910 SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, 911 SCX_ENQ_HEAD = ENQUEUE_HEAD, 912 SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, 913 914 /* high 32bits are SCX specific */ 915 916 /* 917 * Set the following to trigger preemption when calling 918 * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the 919 * current task is cleared to zero and the CPU is kicked into the 920 * scheduling path. Implies %SCX_ENQ_HEAD. 921 */ 922 SCX_ENQ_PREEMPT = 1LLU << 32, 923 924 /* 925 * The task being enqueued was previously enqueued on the current CPU's 926 * %SCX_DSQ_LOCAL, but was removed from it in a call to the 927 * scx_bpf_reenqueue_local() kfunc. If scx_bpf_reenqueue_local() was 928 * invoked in a ->cpu_release() callback, and the task is again 929 * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the 930 * task will not be scheduled on the CPU until at least the next invocation 931 * of the ->cpu_acquire() callback. 932 */ 933 SCX_ENQ_REENQ = 1LLU << 40, 934 935 /* 936 * The task being enqueued is the only task available for the cpu. By 937 * default, ext core keeps executing such tasks but when 938 * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the 939 * %SCX_ENQ_LAST flag set. 940 * 941 * The BPF scheduler is responsible for triggering a follow-up 942 * scheduling event. Otherwise, Execution may stall. 943 */ 944 SCX_ENQ_LAST = 1LLU << 41, 945 946 /* high 8 bits are internal */ 947 __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, 948 949 SCX_ENQ_CLEAR_OPSS = 1LLU << 56, 950 SCX_ENQ_DSQ_PRIQ = 1LLU << 57, 951 }; 952 953 enum scx_deq_flags { 954 /* expose select DEQUEUE_* flags as enums */ 955 SCX_DEQ_SLEEP = DEQUEUE_SLEEP, 956 957 /* high 32bits are SCX specific */ 958 959 /* 960 * The generic core-sched layer decided to execute the task even though 961 * it hasn't been dispatched yet. Dequeue from the BPF side. 962 */ 963 SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, 964 }; 965 966 enum scx_pick_idle_cpu_flags { 967 SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ 968 SCX_PICK_IDLE_IN_NODE = 1LLU << 1, /* pick a CPU in the same target NUMA node */ 969 }; 970 971 enum scx_kick_flags { 972 /* 973 * Kick the target CPU if idle. Guarantees that the target CPU goes 974 * through at least one full scheduling cycle before going idle. If the 975 * target CPU can be determined to be currently not idle and going to go 976 * through a scheduling cycle before going idle, noop. 977 */ 978 SCX_KICK_IDLE = 1LLU << 0, 979 980 /* 981 * Preempt the current task and execute the dispatch path. If the 982 * current task of the target CPU is an SCX task, its ->scx.slice is 983 * cleared to zero before the scheduling path is invoked so that the 984 * task expires and the dispatch path is invoked. 985 */ 986 SCX_KICK_PREEMPT = 1LLU << 1, 987 988 /* 989 * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will 990 * return after the target CPU finishes picking the next task. 991 */ 992 SCX_KICK_WAIT = 1LLU << 2, 993 }; 994 995 enum scx_tg_flags { 996 SCX_TG_ONLINE = 1U << 0, 997 SCX_TG_INITED = 1U << 1, 998 }; 999 1000 enum scx_enable_state { 1001 SCX_ENABLING, 1002 SCX_ENABLED, 1003 SCX_DISABLING, 1004 SCX_DISABLED, 1005 }; 1006 1007 static const char *scx_enable_state_str[] = { 1008 [SCX_ENABLING] = "enabling", 1009 [SCX_ENABLED] = "enabled", 1010 [SCX_DISABLING] = "disabling", 1011 [SCX_DISABLED] = "disabled", 1012 }; 1013 1014 /* 1015 * sched_ext_entity->ops_state 1016 * 1017 * Used to track the task ownership between the SCX core and the BPF scheduler. 1018 * State transitions look as follows: 1019 * 1020 * NONE -> QUEUEING -> QUEUED -> DISPATCHING 1021 * ^ | | 1022 * | v v 1023 * \-------------------------------/ 1024 * 1025 * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call 1026 * sites for explanations on the conditions being waited upon and why they are 1027 * safe. Transitions out of them into NONE or QUEUED must store_release and the 1028 * waiters should load_acquire. 1029 * 1030 * Tracking scx_ops_state enables sched_ext core to reliably determine whether 1031 * any given task can be dispatched by the BPF scheduler at all times and thus 1032 * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler 1033 * to try to dispatch any task anytime regardless of its state as the SCX core 1034 * can safely reject invalid dispatches. 1035 */ 1036 enum scx_ops_state { 1037 SCX_OPSS_NONE, /* owned by the SCX core */ 1038 SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ 1039 SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ 1040 SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ 1041 1042 /* 1043 * QSEQ brands each QUEUED instance so that, when dispatch races 1044 * dequeue/requeue, the dispatcher can tell whether it still has a claim 1045 * on the task being dispatched. 1046 * 1047 * As some 32bit archs can't do 64bit store_release/load_acquire, 1048 * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on 1049 * 32bit machines. The dispatch race window QSEQ protects is very narrow 1050 * and runs with IRQ disabled. 30 bits should be sufficient. 1051 */ 1052 SCX_OPSS_QSEQ_SHIFT = 2, 1053 }; 1054 1055 /* Use macros to ensure that the type is unsigned long for the masks */ 1056 #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) 1057 #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) 1058 1059 DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); 1060 1061 /* 1062 * Return the rq currently locked from an scx callback, or NULL if no rq is 1063 * locked. 1064 */ 1065 static inline struct rq *scx_locked_rq(void) 1066 { 1067 return __this_cpu_read(scx_locked_rq_state); 1068 } 1069 1070 static inline bool scx_kf_allowed_if_unlocked(void) 1071 { 1072 return !current->scx.kf_mask; 1073 } 1074 1075 static inline bool scx_rq_bypassing(struct rq *rq) 1076 { 1077 return unlikely(rq->scx.flags & SCX_RQ_BYPASSING); 1078 } 1079