1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst 4 * 5 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 6 * Copyright (c) 2022 Tejun Heo <tj@kernel.org> 7 * Copyright (c) 2022 David Vernet <dvernet@meta.com> 8 */ 9 #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) 10 11 enum scx_consts { 12 SCX_DSP_DFL_MAX_BATCH = 32, 13 SCX_DSP_MAX_LOOPS = 32, 14 SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ, 15 16 SCX_EXIT_BT_LEN = 64, 17 SCX_EXIT_MSG_LEN = 1024, 18 SCX_EXIT_DUMP_DFL_LEN = 32768, 19 20 SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE, 21 22 /* 23 * Iterating all tasks may take a while. Periodically drop 24 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls. 25 */ 26 SCX_OPS_TASK_ITER_BATCH = 32, 27 }; 28 29 enum scx_exit_kind { 30 SCX_EXIT_NONE, 31 SCX_EXIT_DONE, 32 33 SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */ 34 SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */ 35 SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */ 36 SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */ 37 38 SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */ 39 SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */ 40 SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */ 41 }; 42 43 /* 44 * An exit code can be specified when exiting with scx_bpf_exit() or 45 * scx_ops_exit(), corresponding to exit_kind UNREG_BPF and UNREG_KERN 46 * respectively. The codes are 64bit of the format: 47 * 48 * Bits: [63 .. 48 47 .. 32 31 .. 0] 49 * [ SYS ACT ] [ SYS RSN ] [ USR ] 50 * 51 * SYS ACT: System-defined exit actions 52 * SYS RSN: System-defined exit reasons 53 * USR : User-defined exit codes and reasons 54 * 55 * Using the above, users may communicate intention and context by ORing system 56 * actions and/or system reasons with a user-defined exit code. 57 */ 58 enum scx_exit_code { 59 /* Reasons */ 60 SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, 61 62 /* Actions */ 63 SCX_ECODE_ACT_RESTART = 1LLU << 48, 64 }; 65 66 /* 67 * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is 68 * being disabled. 69 */ 70 struct scx_exit_info { 71 /* %SCX_EXIT_* - broad category of the exit reason */ 72 enum scx_exit_kind kind; 73 74 /* exit code if gracefully exiting */ 75 s64 exit_code; 76 77 /* textual representation of the above */ 78 const char *reason; 79 80 /* backtrace if exiting due to an error */ 81 unsigned long *bt; 82 u32 bt_len; 83 84 /* informational message */ 85 char *msg; 86 87 /* debug dump */ 88 char *dump; 89 }; 90 91 /* sched_ext_ops.flags */ 92 enum scx_ops_flags { 93 /* 94 * Keep built-in idle tracking even if ops.update_idle() is implemented. 95 */ 96 SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, 97 98 /* 99 * By default, if there are no other task to run on the CPU, ext core 100 * keeps running the current task even after its slice expires. If this 101 * flag is specified, such tasks are passed to ops.enqueue() with 102 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info. 103 */ 104 SCX_OPS_ENQ_LAST = 1LLU << 1, 105 106 /* 107 * An exiting task may schedule after PF_EXITING is set. In such cases, 108 * bpf_task_from_pid() may not be able to find the task and if the BPF 109 * scheduler depends on pid lookup for dispatching, the task will be 110 * lost leading to various issues including RCU grace period stalls. 111 * 112 * To mask this problem, by default, unhashed tasks are automatically 113 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't 114 * depend on pid lookups and wants to handle these tasks directly, the 115 * following flag can be used. 116 */ 117 SCX_OPS_ENQ_EXITING = 1LLU << 2, 118 119 /* 120 * If set, only tasks with policy set to SCHED_EXT are attached to 121 * sched_ext. If clear, SCHED_NORMAL tasks are also included. 122 */ 123 SCX_OPS_SWITCH_PARTIAL = 1LLU << 3, 124 125 /* 126 * CPU cgroup support flags 127 */ 128 SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16, /* cpu.weight */ 129 130 SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | 131 SCX_OPS_ENQ_LAST | 132 SCX_OPS_ENQ_EXITING | 133 SCX_OPS_SWITCH_PARTIAL | 134 SCX_OPS_HAS_CGROUP_WEIGHT, 135 }; 136 137 /* argument container for ops.init_task() */ 138 struct scx_init_task_args { 139 /* 140 * Set if ops.init_task() is being invoked on the fork path, as opposed 141 * to the scheduler transition path. 142 */ 143 bool fork; 144 #ifdef CONFIG_EXT_GROUP_SCHED 145 /* the cgroup the task is joining */ 146 struct cgroup *cgroup; 147 #endif 148 }; 149 150 /* argument container for ops.exit_task() */ 151 struct scx_exit_task_args { 152 /* Whether the task exited before running on sched_ext. */ 153 bool cancelled; 154 }; 155 156 /* argument container for ops->cgroup_init() */ 157 struct scx_cgroup_init_args { 158 /* the weight of the cgroup [1..10000] */ 159 u32 weight; 160 }; 161 162 enum scx_cpu_preempt_reason { 163 /* next task is being scheduled by &sched_class_rt */ 164 SCX_CPU_PREEMPT_RT, 165 /* next task is being scheduled by &sched_class_dl */ 166 SCX_CPU_PREEMPT_DL, 167 /* next task is being scheduled by &sched_class_stop */ 168 SCX_CPU_PREEMPT_STOP, 169 /* unknown reason for SCX being preempted */ 170 SCX_CPU_PREEMPT_UNKNOWN, 171 }; 172 173 /* 174 * Argument container for ops->cpu_acquire(). Currently empty, but may be 175 * expanded in the future. 176 */ 177 struct scx_cpu_acquire_args {}; 178 179 /* argument container for ops->cpu_release() */ 180 struct scx_cpu_release_args { 181 /* the reason the CPU was preempted */ 182 enum scx_cpu_preempt_reason reason; 183 184 /* the task that's going to be scheduled on the CPU */ 185 struct task_struct *task; 186 }; 187 188 /* 189 * Informational context provided to dump operations. 190 */ 191 struct scx_dump_ctx { 192 enum scx_exit_kind kind; 193 s64 exit_code; 194 const char *reason; 195 u64 at_ns; 196 u64 at_jiffies; 197 }; 198 199 /** 200 * struct sched_ext_ops - Operation table for BPF scheduler implementation 201 * 202 * Userland can implement an arbitrary scheduling policy by implementing and 203 * loading operations in this table. 204 */ 205 struct sched_ext_ops { 206 /** 207 * select_cpu - Pick the target CPU for a task which is being woken up 208 * @p: task being woken up 209 * @prev_cpu: the cpu @p was on before sleeping 210 * @wake_flags: SCX_WAKE_* 211 * 212 * Decision made here isn't final. @p may be moved to any CPU while it 213 * is getting dispatched for execution later. However, as @p is not on 214 * the rq at this point, getting the eventual execution CPU right here 215 * saves a small bit of overhead down the line. 216 * 217 * If an idle CPU is returned, the CPU is kicked and will try to 218 * dispatch. While an explicit custom mechanism can be added, 219 * select_cpu() serves as the default way to wake up idle CPUs. 220 * 221 * @p may be dispatched directly by calling scx_bpf_dispatch(). If @p 222 * is dispatched, the ops.enqueue() callback will be skipped. Finally, 223 * if @p is dispatched to SCX_DSQ_LOCAL, it will be dispatched to the 224 * local DSQ of whatever CPU is returned by this callback. 225 */ 226 s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 227 228 /** 229 * enqueue - Enqueue a task on the BPF scheduler 230 * @p: task being enqueued 231 * @enq_flags: %SCX_ENQ_* 232 * 233 * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch() 234 * or enqueue on the BPF scheduler. If not directly dispatched, the bpf 235 * scheduler owns @p and if it fails to dispatch @p, the task will 236 * stall. 237 * 238 * If @p was dispatched from ops.select_cpu(), this callback is 239 * skipped. 240 */ 241 void (*enqueue)(struct task_struct *p, u64 enq_flags); 242 243 /** 244 * dequeue - Remove a task from the BPF scheduler 245 * @p: task being dequeued 246 * @deq_flags: %SCX_DEQ_* 247 * 248 * Remove @p from the BPF scheduler. This is usually called to isolate 249 * the task while updating its scheduling properties (e.g. priority). 250 * 251 * The ext core keeps track of whether the BPF side owns a given task or 252 * not and can gracefully ignore spurious dispatches from BPF side, 253 * which makes it safe to not implement this method. However, depending 254 * on the scheduling logic, this can lead to confusing behaviors - e.g. 255 * scheduling position not being updated across a priority change. 256 */ 257 void (*dequeue)(struct task_struct *p, u64 deq_flags); 258 259 /** 260 * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs 261 * @cpu: CPU to dispatch tasks for 262 * @prev: previous task being switched out 263 * 264 * Called when a CPU's local dsq is empty. The operation should dispatch 265 * one or more tasks from the BPF scheduler into the DSQs using 266 * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using 267 * scx_bpf_consume(). 268 * 269 * The maximum number of times scx_bpf_dispatch() can be called without 270 * an intervening scx_bpf_consume() is specified by 271 * ops.dispatch_max_batch. See the comments on top of the two functions 272 * for more details. 273 * 274 * When not %NULL, @prev is an SCX task with its slice depleted. If 275 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in 276 * @prev->scx.flags, it is not enqueued yet and will be enqueued after 277 * ops.dispatch() returns. To keep executing @prev, return without 278 * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST. 279 */ 280 void (*dispatch)(s32 cpu, struct task_struct *prev); 281 282 /** 283 * tick - Periodic tick 284 * @p: task running currently 285 * 286 * This operation is called every 1/HZ seconds on CPUs which are 287 * executing an SCX task. Setting @p->scx.slice to 0 will trigger an 288 * immediate dispatch cycle on the CPU. 289 */ 290 void (*tick)(struct task_struct *p); 291 292 /** 293 * runnable - A task is becoming runnable on its associated CPU 294 * @p: task becoming runnable 295 * @enq_flags: %SCX_ENQ_* 296 * 297 * This and the following three functions can be used to track a task's 298 * execution state transitions. A task becomes ->runnable() on a CPU, 299 * and then goes through one or more ->running() and ->stopping() pairs 300 * as it runs on the CPU, and eventually becomes ->quiescent() when it's 301 * done running on the CPU. 302 * 303 * @p is becoming runnable on the CPU because it's 304 * 305 * - waking up (%SCX_ENQ_WAKEUP) 306 * - being moved from another CPU 307 * - being restored after temporarily taken off the queue for an 308 * attribute change. 309 * 310 * This and ->enqueue() are related but not coupled. This operation 311 * notifies @p's state transition and may not be followed by ->enqueue() 312 * e.g. when @p is being dispatched to a remote CPU, or when @p is 313 * being enqueued on a CPU experiencing a hotplug event. Likewise, a 314 * task may be ->enqueue()'d without being preceded by this operation 315 * e.g. after exhausting its slice. 316 */ 317 void (*runnable)(struct task_struct *p, u64 enq_flags); 318 319 /** 320 * running - A task is starting to run on its associated CPU 321 * @p: task starting to run 322 * 323 * See ->runnable() for explanation on the task state notifiers. 324 */ 325 void (*running)(struct task_struct *p); 326 327 /** 328 * stopping - A task is stopping execution 329 * @p: task stopping to run 330 * @runnable: is task @p still runnable? 331 * 332 * See ->runnable() for explanation on the task state notifiers. If 333 * !@runnable, ->quiescent() will be invoked after this operation 334 * returns. 335 */ 336 void (*stopping)(struct task_struct *p, bool runnable); 337 338 /** 339 * quiescent - A task is becoming not runnable on its associated CPU 340 * @p: task becoming not runnable 341 * @deq_flags: %SCX_DEQ_* 342 * 343 * See ->runnable() for explanation on the task state notifiers. 344 * 345 * @p is becoming quiescent on the CPU because it's 346 * 347 * - sleeping (%SCX_DEQ_SLEEP) 348 * - being moved to another CPU 349 * - being temporarily taken off the queue for an attribute change 350 * (%SCX_DEQ_SAVE) 351 * 352 * This and ->dequeue() are related but not coupled. This operation 353 * notifies @p's state transition and may not be preceded by ->dequeue() 354 * e.g. when @p is being dispatched to a remote CPU. 355 */ 356 void (*quiescent)(struct task_struct *p, u64 deq_flags); 357 358 /** 359 * yield - Yield CPU 360 * @from: yielding task 361 * @to: optional yield target task 362 * 363 * If @to is NULL, @from is yielding the CPU to other runnable tasks. 364 * The BPF scheduler should ensure that other available tasks are 365 * dispatched before the yielding task. Return value is ignored in this 366 * case. 367 * 368 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf 369 * scheduler can implement the request, return %true; otherwise, %false. 370 */ 371 bool (*yield)(struct task_struct *from, struct task_struct *to); 372 373 /** 374 * core_sched_before - Task ordering for core-sched 375 * @a: task A 376 * @b: task B 377 * 378 * Used by core-sched to determine the ordering between two tasks. See 379 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on 380 * core-sched. 381 * 382 * Both @a and @b are runnable and may or may not currently be queued on 383 * the BPF scheduler. Should return %true if @a should run before @b. 384 * %false if there's no required ordering or @b should run before @a. 385 * 386 * If not specified, the default is ordering them according to when they 387 * became runnable. 388 */ 389 bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); 390 391 /** 392 * set_weight - Set task weight 393 * @p: task to set weight for 394 * @weight: new weight [1..10000] 395 * 396 * Update @p's weight to @weight. 397 */ 398 void (*set_weight)(struct task_struct *p, u32 weight); 399 400 /** 401 * set_cpumask - Set CPU affinity 402 * @p: task to set CPU affinity for 403 * @cpumask: cpumask of cpus that @p can run on 404 * 405 * Update @p's CPU affinity to @cpumask. 406 */ 407 void (*set_cpumask)(struct task_struct *p, 408 const struct cpumask *cpumask); 409 410 /** 411 * update_idle - Update the idle state of a CPU 412 * @cpu: CPU to udpate the idle state for 413 * @idle: whether entering or exiting the idle state 414 * 415 * This operation is called when @rq's CPU goes or leaves the idle 416 * state. By default, implementing this operation disables the built-in 417 * idle CPU tracking and the following helpers become unavailable: 418 * 419 * - scx_bpf_select_cpu_dfl() 420 * - scx_bpf_test_and_clear_cpu_idle() 421 * - scx_bpf_pick_idle_cpu() 422 * 423 * The user also must implement ops.select_cpu() as the default 424 * implementation relies on scx_bpf_select_cpu_dfl(). 425 * 426 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle 427 * tracking. 428 */ 429 void (*update_idle)(s32 cpu, bool idle); 430 431 /** 432 * cpu_acquire - A CPU is becoming available to the BPF scheduler 433 * @cpu: The CPU being acquired by the BPF scheduler. 434 * @args: Acquire arguments, see the struct definition. 435 * 436 * A CPU that was previously released from the BPF scheduler is now once 437 * again under its control. 438 */ 439 void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); 440 441 /** 442 * cpu_release - A CPU is taken away from the BPF scheduler 443 * @cpu: The CPU being released by the BPF scheduler. 444 * @args: Release arguments, see the struct definition. 445 * 446 * The specified CPU is no longer under the control of the BPF 447 * scheduler. This could be because it was preempted by a higher 448 * priority sched_class, though there may be other reasons as well. The 449 * caller should consult @args->reason to determine the cause. 450 */ 451 void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); 452 453 /** 454 * init_task - Initialize a task to run in a BPF scheduler 455 * @p: task to initialize for BPF scheduling 456 * @args: init arguments, see the struct definition 457 * 458 * Either we're loading a BPF scheduler or a new task is being forked. 459 * Initialize @p for BPF scheduling. This operation may block and can 460 * be used for allocations, and is called exactly once for a task. 461 * 462 * Return 0 for success, -errno for failure. An error return while 463 * loading will abort loading of the BPF scheduler. During a fork, it 464 * will abort that specific fork. 465 */ 466 s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); 467 468 /** 469 * exit_task - Exit a previously-running task from the system 470 * @p: task to exit 471 * 472 * @p is exiting or the BPF scheduler is being unloaded. Perform any 473 * necessary cleanup for @p. 474 */ 475 void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); 476 477 /** 478 * enable - Enable BPF scheduling for a task 479 * @p: task to enable BPF scheduling for 480 * 481 * Enable @p for BPF scheduling. enable() is called on @p any time it 482 * enters SCX, and is always paired with a matching disable(). 483 */ 484 void (*enable)(struct task_struct *p); 485 486 /** 487 * disable - Disable BPF scheduling for a task 488 * @p: task to disable BPF scheduling for 489 * 490 * @p is exiting, leaving SCX or the BPF scheduler is being unloaded. 491 * Disable BPF scheduling for @p. A disable() call is always matched 492 * with a prior enable() call. 493 */ 494 void (*disable)(struct task_struct *p); 495 496 /** 497 * dump - Dump BPF scheduler state on error 498 * @ctx: debug dump context 499 * 500 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump. 501 */ 502 void (*dump)(struct scx_dump_ctx *ctx); 503 504 /** 505 * dump_cpu - Dump BPF scheduler state for a CPU on error 506 * @ctx: debug dump context 507 * @cpu: CPU to generate debug dump for 508 * @idle: @cpu is currently idle without any runnable tasks 509 * 510 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 511 * @cpu. If @idle is %true and this operation doesn't produce any 512 * output, @cpu is skipped for dump. 513 */ 514 void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle); 515 516 /** 517 * dump_task - Dump BPF scheduler state for a runnable task on error 518 * @ctx: debug dump context 519 * @p: runnable task to generate debug dump for 520 * 521 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for 522 * @p. 523 */ 524 void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p); 525 526 #ifdef CONFIG_EXT_GROUP_SCHED 527 /** 528 * cgroup_init - Initialize a cgroup 529 * @cgrp: cgroup being initialized 530 * @args: init arguments, see the struct definition 531 * 532 * Either the BPF scheduler is being loaded or @cgrp created, initialize 533 * @cgrp for sched_ext. This operation may block. 534 * 535 * Return 0 for success, -errno for failure. An error return while 536 * loading will abort loading of the BPF scheduler. During cgroup 537 * creation, it will abort the specific cgroup creation. 538 */ 539 s32 (*cgroup_init)(struct cgroup *cgrp, 540 struct scx_cgroup_init_args *args); 541 542 /** 543 * cgroup_exit - Exit a cgroup 544 * @cgrp: cgroup being exited 545 * 546 * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit 547 * @cgrp for sched_ext. This operation my block. 548 */ 549 void (*cgroup_exit)(struct cgroup *cgrp); 550 551 /** 552 * cgroup_prep_move - Prepare a task to be moved to a different cgroup 553 * @p: task being moved 554 * @from: cgroup @p is being moved from 555 * @to: cgroup @p is being moved to 556 * 557 * Prepare @p for move from cgroup @from to @to. This operation may 558 * block and can be used for allocations. 559 * 560 * Return 0 for success, -errno for failure. An error return aborts the 561 * migration. 562 */ 563 s32 (*cgroup_prep_move)(struct task_struct *p, 564 struct cgroup *from, struct cgroup *to); 565 566 /** 567 * cgroup_move - Commit cgroup move 568 * @p: task being moved 569 * @from: cgroup @p is being moved from 570 * @to: cgroup @p is being moved to 571 * 572 * Commit the move. @p is dequeued during this operation. 573 */ 574 void (*cgroup_move)(struct task_struct *p, 575 struct cgroup *from, struct cgroup *to); 576 577 /** 578 * cgroup_cancel_move - Cancel cgroup move 579 * @p: task whose cgroup move is being canceled 580 * @from: cgroup @p was being moved from 581 * @to: cgroup @p was being moved to 582 * 583 * @p was cgroup_prep_move()'d but failed before reaching cgroup_move(). 584 * Undo the preparation. 585 */ 586 void (*cgroup_cancel_move)(struct task_struct *p, 587 struct cgroup *from, struct cgroup *to); 588 589 /** 590 * cgroup_set_weight - A cgroup's weight is being changed 591 * @cgrp: cgroup whose weight is being updated 592 * @weight: new weight [1..10000] 593 * 594 * Update @tg's weight to @weight. 595 */ 596 void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight); 597 #endif /* CONFIG_CGROUPS */ 598 599 /* 600 * All online ops must come before ops.cpu_online(). 601 */ 602 603 /** 604 * cpu_online - A CPU became online 605 * @cpu: CPU which just came up 606 * 607 * @cpu just came online. @cpu will not call ops.enqueue() or 608 * ops.dispatch(), nor run tasks associated with other CPUs beforehand. 609 */ 610 void (*cpu_online)(s32 cpu); 611 612 /** 613 * cpu_offline - A CPU is going offline 614 * @cpu: CPU which is going offline 615 * 616 * @cpu is going offline. @cpu will not call ops.enqueue() or 617 * ops.dispatch(), nor run tasks associated with other CPUs afterwards. 618 */ 619 void (*cpu_offline)(s32 cpu); 620 621 /* 622 * All CPU hotplug ops must come before ops.init(). 623 */ 624 625 /** 626 * init - Initialize the BPF scheduler 627 */ 628 s32 (*init)(void); 629 630 /** 631 * exit - Clean up after the BPF scheduler 632 * @info: Exit info 633 * 634 * ops.exit() is also called on ops.init() failure, which is a bit 635 * unusual. This is to allow rich reporting through @info on how 636 * ops.init() failed. 637 */ 638 void (*exit)(struct scx_exit_info *info); 639 640 /** 641 * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch 642 */ 643 u32 dispatch_max_batch; 644 645 /** 646 * flags - %SCX_OPS_* flags 647 */ 648 u64 flags; 649 650 /** 651 * timeout_ms - The maximum amount of time, in milliseconds, that a 652 * runnable task should be able to wait before being scheduled. The 653 * maximum timeout may not exceed the default timeout of 30 seconds. 654 * 655 * Defaults to the maximum allowed timeout value of 30 seconds. 656 */ 657 u32 timeout_ms; 658 659 /** 660 * exit_dump_len - scx_exit_info.dump buffer length. If 0, the default 661 * value of 32768 is used. 662 */ 663 u32 exit_dump_len; 664 665 /** 666 * hotplug_seq - A sequence number that may be set by the scheduler to 667 * detect when a hotplug event has occurred during the loading process. 668 * If 0, no detection occurs. Otherwise, the scheduler will fail to 669 * load if the sequence number does not match @scx_hotplug_seq on the 670 * enable path. 671 */ 672 u64 hotplug_seq; 673 674 /** 675 * name - BPF scheduler's name 676 * 677 * Must be a non-zero valid BPF object name including only isalnum(), 678 * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the 679 * BPF scheduler is enabled. 680 */ 681 char name[SCX_OPS_NAME_LEN]; 682 }; 683 684 enum scx_opi { 685 SCX_OPI_BEGIN = 0, 686 SCX_OPI_NORMAL_BEGIN = 0, 687 SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online), 688 SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online), 689 SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init), 690 SCX_OPI_END = SCX_OP_IDX(init), 691 }; 692 693 enum scx_wake_flags { 694 /* expose select WF_* flags as enums */ 695 SCX_WAKE_FORK = WF_FORK, 696 SCX_WAKE_TTWU = WF_TTWU, 697 SCX_WAKE_SYNC = WF_SYNC, 698 }; 699 700 enum scx_enq_flags { 701 /* expose select ENQUEUE_* flags as enums */ 702 SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP, 703 SCX_ENQ_HEAD = ENQUEUE_HEAD, 704 SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED, 705 706 /* high 32bits are SCX specific */ 707 708 /* 709 * Set the following to trigger preemption when calling 710 * scx_bpf_dispatch() with a local dsq as the target. The slice of the 711 * current task is cleared to zero and the CPU is kicked into the 712 * scheduling path. Implies %SCX_ENQ_HEAD. 713 */ 714 SCX_ENQ_PREEMPT = 1LLU << 32, 715 716 /* 717 * The task being enqueued was previously enqueued on the current CPU's 718 * %SCX_DSQ_LOCAL, but was removed from it in a call to the 719 * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was 720 * invoked in a ->cpu_release() callback, and the task is again 721 * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the 722 * task will not be scheduled on the CPU until at least the next invocation 723 * of the ->cpu_acquire() callback. 724 */ 725 SCX_ENQ_REENQ = 1LLU << 40, 726 727 /* 728 * The task being enqueued is the only task available for the cpu. By 729 * default, ext core keeps executing such tasks but when 730 * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the 731 * %SCX_ENQ_LAST flag set. 732 * 733 * The BPF scheduler is responsible for triggering a follow-up 734 * scheduling event. Otherwise, Execution may stall. 735 */ 736 SCX_ENQ_LAST = 1LLU << 41, 737 738 /* high 8 bits are internal */ 739 __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, 740 741 SCX_ENQ_CLEAR_OPSS = 1LLU << 56, 742 SCX_ENQ_DSQ_PRIQ = 1LLU << 57, 743 }; 744 745 enum scx_deq_flags { 746 /* expose select DEQUEUE_* flags as enums */ 747 SCX_DEQ_SLEEP = DEQUEUE_SLEEP, 748 749 /* high 32bits are SCX specific */ 750 751 /* 752 * The generic core-sched layer decided to execute the task even though 753 * it hasn't been dispatched yet. Dequeue from the BPF side. 754 */ 755 SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32, 756 }; 757 758 enum scx_pick_idle_cpu_flags { 759 SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */ 760 }; 761 762 enum scx_kick_flags { 763 /* 764 * Kick the target CPU if idle. Guarantees that the target CPU goes 765 * through at least one full scheduling cycle before going idle. If the 766 * target CPU can be determined to be currently not idle and going to go 767 * through a scheduling cycle before going idle, noop. 768 */ 769 SCX_KICK_IDLE = 1LLU << 0, 770 771 /* 772 * Preempt the current task and execute the dispatch path. If the 773 * current task of the target CPU is an SCX task, its ->scx.slice is 774 * cleared to zero before the scheduling path is invoked so that the 775 * task expires and the dispatch path is invoked. 776 */ 777 SCX_KICK_PREEMPT = 1LLU << 1, 778 779 /* 780 * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will 781 * return after the target CPU finishes picking the next task. 782 */ 783 SCX_KICK_WAIT = 1LLU << 2, 784 }; 785 786 enum scx_tg_flags { 787 SCX_TG_ONLINE = 1U << 0, 788 SCX_TG_INITED = 1U << 1, 789 }; 790 791 enum scx_ops_enable_state { 792 SCX_OPS_ENABLING, 793 SCX_OPS_ENABLED, 794 SCX_OPS_DISABLING, 795 SCX_OPS_DISABLED, 796 }; 797 798 static const char *scx_ops_enable_state_str[] = { 799 [SCX_OPS_ENABLING] = "enabling", 800 [SCX_OPS_ENABLED] = "enabled", 801 [SCX_OPS_DISABLING] = "disabling", 802 [SCX_OPS_DISABLED] = "disabled", 803 }; 804 805 /* 806 * sched_ext_entity->ops_state 807 * 808 * Used to track the task ownership between the SCX core and the BPF scheduler. 809 * State transitions look as follows: 810 * 811 * NONE -> QUEUEING -> QUEUED -> DISPATCHING 812 * ^ | | 813 * | v v 814 * \-------------------------------/ 815 * 816 * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call 817 * sites for explanations on the conditions being waited upon and why they are 818 * safe. Transitions out of them into NONE or QUEUED must store_release and the 819 * waiters should load_acquire. 820 * 821 * Tracking scx_ops_state enables sched_ext core to reliably determine whether 822 * any given task can be dispatched by the BPF scheduler at all times and thus 823 * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler 824 * to try to dispatch any task anytime regardless of its state as the SCX core 825 * can safely reject invalid dispatches. 826 */ 827 enum scx_ops_state { 828 SCX_OPSS_NONE, /* owned by the SCX core */ 829 SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */ 830 SCX_OPSS_QUEUED, /* owned by the BPF scheduler */ 831 SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */ 832 833 /* 834 * QSEQ brands each QUEUED instance so that, when dispatch races 835 * dequeue/requeue, the dispatcher can tell whether it still has a claim 836 * on the task being dispatched. 837 * 838 * As some 32bit archs can't do 64bit store_release/load_acquire, 839 * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on 840 * 32bit machines. The dispatch race window QSEQ protects is very narrow 841 * and runs with IRQ disabled. 30 bits should be sufficient. 842 */ 843 SCX_OPSS_QSEQ_SHIFT = 2, 844 }; 845 846 /* Use macros to ensure that the type is unsigned long for the masks */ 847 #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1) 848 #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK) 849 850 /* 851 * During exit, a task may schedule after losing its PIDs. When disabling the 852 * BPF scheduler, we need to be able to iterate tasks in every state to 853 * guarantee system safety. Maintain a dedicated task list which contains every 854 * task between its fork and eventual free. 855 */ 856 static DEFINE_SPINLOCK(scx_tasks_lock); 857 static LIST_HEAD(scx_tasks); 858 859 /* ops enable/disable */ 860 static struct kthread_worker *scx_ops_helper; 861 static DEFINE_MUTEX(scx_ops_enable_mutex); 862 DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled); 863 DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); 864 static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED); 865 static atomic_t scx_ops_bypass_depth = ATOMIC_INIT(0); 866 static bool scx_ops_init_task_enabled; 867 static bool scx_switching_all; 868 DEFINE_STATIC_KEY_FALSE(__scx_switched_all); 869 870 static struct sched_ext_ops scx_ops; 871 static bool scx_warned_zero_slice; 872 873 static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last); 874 static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting); 875 static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt); 876 static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled); 877 878 static struct static_key_false scx_has_op[SCX_OPI_END] = 879 { [0 ... SCX_OPI_END-1] = STATIC_KEY_FALSE_INIT }; 880 881 static atomic_t scx_exit_kind = ATOMIC_INIT(SCX_EXIT_DONE); 882 static struct scx_exit_info *scx_exit_info; 883 884 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); 885 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); 886 887 /* 888 * A monotically increasing sequence number that is incremented every time a 889 * scheduler is enabled. This can be used by to check if any custom sched_ext 890 * scheduler has ever been used in the system. 891 */ 892 static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); 893 894 /* 895 * The maximum amount of time in jiffies that a task may be runnable without 896 * being scheduled on a CPU. If this timeout is exceeded, it will trigger 897 * scx_ops_error(). 898 */ 899 static unsigned long scx_watchdog_timeout; 900 901 /* 902 * The last time the delayed work was run. This delayed work relies on 903 * ksoftirqd being able to run to service timer interrupts, so it's possible 904 * that this work itself could get wedged. To account for this, we check that 905 * it's not stalled in the timer tick, and trigger an error if it is. 906 */ 907 static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; 908 909 static struct delayed_work scx_watchdog_work; 910 911 /* idle tracking */ 912 #ifdef CONFIG_SMP 913 #ifdef CONFIG_CPUMASK_OFFSTACK 914 #define CL_ALIGNED_IF_ONSTACK 915 #else 916 #define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp 917 #endif 918 919 static struct { 920 cpumask_var_t cpu; 921 cpumask_var_t smt; 922 } idle_masks CL_ALIGNED_IF_ONSTACK; 923 924 #endif /* CONFIG_SMP */ 925 926 /* for %SCX_KICK_WAIT */ 927 static unsigned long __percpu *scx_kick_cpus_pnt_seqs; 928 929 /* 930 * Direct dispatch marker. 931 * 932 * Non-NULL values are used for direct dispatch from enqueue path. A valid 933 * pointer points to the task currently being enqueued. An ERR_PTR value is used 934 * to indicate that direct dispatch has already happened. 935 */ 936 static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); 937 938 /* 939 * Dispatch queues. 940 * 941 * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. This is 942 * to avoid live-locking in bypass mode where all tasks are dispatched to 943 * %SCX_DSQ_GLOBAL and all CPUs consume from it. If per-node split isn't 944 * sufficient, it can be further split. 945 */ 946 static struct scx_dispatch_q **global_dsqs; 947 948 static const struct rhashtable_params dsq_hash_params = { 949 .key_len = 8, 950 .key_offset = offsetof(struct scx_dispatch_q, id), 951 .head_offset = offsetof(struct scx_dispatch_q, hash_node), 952 }; 953 954 static struct rhashtable dsq_hash; 955 static LLIST_HEAD(dsqs_to_free); 956 957 /* dispatch buf */ 958 struct scx_dsp_buf_ent { 959 struct task_struct *task; 960 unsigned long qseq; 961 u64 dsq_id; 962 u64 enq_flags; 963 }; 964 965 static u32 scx_dsp_max_batch; 966 967 struct scx_dsp_ctx { 968 struct rq *rq; 969 u32 cursor; 970 u32 nr_tasks; 971 struct scx_dsp_buf_ent buf[]; 972 }; 973 974 static struct scx_dsp_ctx __percpu *scx_dsp_ctx; 975 976 /* string formatting from BPF */ 977 struct scx_bstr_buf { 978 u64 data[MAX_BPRINTF_VARARGS]; 979 char line[SCX_EXIT_MSG_LEN]; 980 }; 981 982 static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); 983 static struct scx_bstr_buf scx_exit_bstr_buf; 984 985 /* ops debug dump */ 986 struct scx_dump_data { 987 s32 cpu; 988 bool first; 989 s32 cursor; 990 struct seq_buf *s; 991 const char *prefix; 992 struct scx_bstr_buf buf; 993 }; 994 995 static struct scx_dump_data scx_dump_data = { 996 .cpu = -1, 997 }; 998 999 /* /sys/kernel/sched_ext interface */ 1000 static struct kset *scx_kset; 1001 static struct kobject *scx_root_kobj; 1002 1003 #define CREATE_TRACE_POINTS 1004 #include <trace/events/sched_ext.h> 1005 1006 static void process_ddsp_deferred_locals(struct rq *rq); 1007 static void scx_bpf_kick_cpu(s32 cpu, u64 flags); 1008 static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind, 1009 s64 exit_code, 1010 const char *fmt, ...); 1011 1012 #define scx_ops_error_kind(err, fmt, args...) \ 1013 scx_ops_exit_kind((err), 0, fmt, ##args) 1014 1015 #define scx_ops_exit(code, fmt, args...) \ 1016 scx_ops_exit_kind(SCX_EXIT_UNREG_KERN, (code), fmt, ##args) 1017 1018 #define scx_ops_error(fmt, args...) \ 1019 scx_ops_error_kind(SCX_EXIT_ERROR, fmt, ##args) 1020 1021 #define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)]) 1022 1023 static long jiffies_delta_msecs(unsigned long at, unsigned long now) 1024 { 1025 if (time_after(at, now)) 1026 return jiffies_to_msecs(at - now); 1027 else 1028 return -(long)jiffies_to_msecs(now - at); 1029 } 1030 1031 /* if the highest set bit is N, return a mask with bits [N+1, 31] set */ 1032 static u32 higher_bits(u32 flags) 1033 { 1034 return ~((1 << fls(flags)) - 1); 1035 } 1036 1037 /* return the mask with only the highest bit set */ 1038 static u32 highest_bit(u32 flags) 1039 { 1040 int bit = fls(flags); 1041 return ((u64)1 << bit) >> 1; 1042 } 1043 1044 static bool u32_before(u32 a, u32 b) 1045 { 1046 return (s32)(a - b) < 0; 1047 } 1048 1049 static struct scx_dispatch_q *find_global_dsq(struct task_struct *p) 1050 { 1051 return global_dsqs[cpu_to_node(task_cpu(p))]; 1052 } 1053 1054 static struct scx_dispatch_q *find_user_dsq(u64 dsq_id) 1055 { 1056 return rhashtable_lookup_fast(&dsq_hash, &dsq_id, dsq_hash_params); 1057 } 1058 1059 /* 1060 * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX 1061 * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate 1062 * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check 1063 * whether it's running from an allowed context. 1064 * 1065 * @mask is constant, always inline to cull the mask calculations. 1066 */ 1067 static __always_inline void scx_kf_allow(u32 mask) 1068 { 1069 /* nesting is allowed only in increasing scx_kf_mask order */ 1070 WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask, 1071 "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n", 1072 current->scx.kf_mask, mask); 1073 current->scx.kf_mask |= mask; 1074 barrier(); 1075 } 1076 1077 static void scx_kf_disallow(u32 mask) 1078 { 1079 barrier(); 1080 current->scx.kf_mask &= ~mask; 1081 } 1082 1083 #define SCX_CALL_OP(mask, op, args...) \ 1084 do { \ 1085 if (mask) { \ 1086 scx_kf_allow(mask); \ 1087 scx_ops.op(args); \ 1088 scx_kf_disallow(mask); \ 1089 } else { \ 1090 scx_ops.op(args); \ 1091 } \ 1092 } while (0) 1093 1094 #define SCX_CALL_OP_RET(mask, op, args...) \ 1095 ({ \ 1096 __typeof__(scx_ops.op(args)) __ret; \ 1097 if (mask) { \ 1098 scx_kf_allow(mask); \ 1099 __ret = scx_ops.op(args); \ 1100 scx_kf_disallow(mask); \ 1101 } else { \ 1102 __ret = scx_ops.op(args); \ 1103 } \ 1104 __ret; \ 1105 }) 1106 1107 /* 1108 * Some kfuncs are allowed only on the tasks that are subjects of the 1109 * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such 1110 * restrictions, the following SCX_CALL_OP_*() variants should be used when 1111 * invoking scx_ops operations that take task arguments. These can only be used 1112 * for non-nesting operations due to the way the tasks are tracked. 1113 * 1114 * kfuncs which can only operate on such tasks can in turn use 1115 * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on 1116 * the specific task. 1117 */ 1118 #define SCX_CALL_OP_TASK(mask, op, task, args...) \ 1119 do { \ 1120 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 1121 current->scx.kf_tasks[0] = task; \ 1122 SCX_CALL_OP(mask, op, task, ##args); \ 1123 current->scx.kf_tasks[0] = NULL; \ 1124 } while (0) 1125 1126 #define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \ 1127 ({ \ 1128 __typeof__(scx_ops.op(task, ##args)) __ret; \ 1129 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 1130 current->scx.kf_tasks[0] = task; \ 1131 __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \ 1132 current->scx.kf_tasks[0] = NULL; \ 1133 __ret; \ 1134 }) 1135 1136 #define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \ 1137 ({ \ 1138 __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \ 1139 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ 1140 current->scx.kf_tasks[0] = task0; \ 1141 current->scx.kf_tasks[1] = task1; \ 1142 __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \ 1143 current->scx.kf_tasks[0] = NULL; \ 1144 current->scx.kf_tasks[1] = NULL; \ 1145 __ret; \ 1146 }) 1147 1148 /* @mask is constant, always inline to cull unnecessary branches */ 1149 static __always_inline bool scx_kf_allowed(u32 mask) 1150 { 1151 if (unlikely(!(current->scx.kf_mask & mask))) { 1152 scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x", 1153 mask, current->scx.kf_mask); 1154 return false; 1155 } 1156 1157 /* 1158 * Enforce nesting boundaries. e.g. A kfunc which can be called from 1159 * DISPATCH must not be called if we're running DEQUEUE which is nested 1160 * inside ops.dispatch(). We don't need to check boundaries for any 1161 * blocking kfuncs as the verifier ensures they're only called from 1162 * sleepable progs. 1163 */ 1164 if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && 1165 (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { 1166 scx_ops_error("cpu_release kfunc called from a nested operation"); 1167 return false; 1168 } 1169 1170 if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && 1171 (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) { 1172 scx_ops_error("dispatch kfunc called from a nested operation"); 1173 return false; 1174 } 1175 1176 return true; 1177 } 1178 1179 /* see SCX_CALL_OP_TASK() */ 1180 static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask, 1181 struct task_struct *p) 1182 { 1183 if (!scx_kf_allowed(mask)) 1184 return false; 1185 1186 if (unlikely((p != current->scx.kf_tasks[0] && 1187 p != current->scx.kf_tasks[1]))) { 1188 scx_ops_error("called on a task not being operated on"); 1189 return false; 1190 } 1191 1192 return true; 1193 } 1194 1195 static bool scx_kf_allowed_if_unlocked(void) 1196 { 1197 return !current->scx.kf_mask; 1198 } 1199 1200 /** 1201 * nldsq_next_task - Iterate to the next task in a non-local DSQ 1202 * @dsq: user dsq being interated 1203 * @cur: current position, %NULL to start iteration 1204 * @rev: walk backwards 1205 * 1206 * Returns %NULL when iteration is finished. 1207 */ 1208 static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, 1209 struct task_struct *cur, bool rev) 1210 { 1211 struct list_head *list_node; 1212 struct scx_dsq_list_node *dsq_lnode; 1213 1214 lockdep_assert_held(&dsq->lock); 1215 1216 if (cur) 1217 list_node = &cur->scx.dsq_list.node; 1218 else 1219 list_node = &dsq->list; 1220 1221 /* find the next task, need to skip BPF iteration cursors */ 1222 do { 1223 if (rev) 1224 list_node = list_node->prev; 1225 else 1226 list_node = list_node->next; 1227 1228 if (list_node == &dsq->list) 1229 return NULL; 1230 1231 dsq_lnode = container_of(list_node, struct scx_dsq_list_node, 1232 node); 1233 } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); 1234 1235 return container_of(dsq_lnode, struct task_struct, scx.dsq_list); 1236 } 1237 1238 #define nldsq_for_each_task(p, dsq) \ 1239 for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ 1240 (p) = nldsq_next_task((dsq), (p), false)) 1241 1242 1243 /* 1244 * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] 1245 * dispatch order. BPF-visible iterator is opaque and larger to allow future 1246 * changes without breaking backward compatibility. Can be used with 1247 * bpf_for_each(). See bpf_iter_scx_dsq_*(). 1248 */ 1249 enum scx_dsq_iter_flags { 1250 /* iterate in the reverse dispatch order */ 1251 SCX_DSQ_ITER_REV = 1U << 16, 1252 1253 __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, 1254 __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, 1255 1256 __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, 1257 __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | 1258 __SCX_DSQ_ITER_HAS_SLICE | 1259 __SCX_DSQ_ITER_HAS_VTIME, 1260 }; 1261 1262 struct bpf_iter_scx_dsq_kern { 1263 struct scx_dsq_list_node cursor; 1264 struct scx_dispatch_q *dsq; 1265 u64 slice; 1266 u64 vtime; 1267 } __attribute__((aligned(8))); 1268 1269 struct bpf_iter_scx_dsq { 1270 u64 __opaque[6]; 1271 } __attribute__((aligned(8))); 1272 1273 1274 /* 1275 * SCX task iterator. 1276 */ 1277 struct scx_task_iter { 1278 struct sched_ext_entity cursor; 1279 struct task_struct *locked; 1280 struct rq *rq; 1281 struct rq_flags rf; 1282 u32 cnt; 1283 }; 1284 1285 /** 1286 * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration 1287 * @iter: iterator to init 1288 * 1289 * Initialize @iter and return with scx_tasks_lock held. Once initialized, @iter 1290 * must eventually be stopped with scx_task_iter_stop(). 1291 * 1292 * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock() 1293 * between this and the first next() call or between any two next() calls. If 1294 * the locks are released between two next() calls, the caller is responsible 1295 * for ensuring that the task being iterated remains accessible either through 1296 * RCU read lock or obtaining a reference count. 1297 * 1298 * All tasks which existed when the iteration started are guaranteed to be 1299 * visited as long as they still exist. 1300 */ 1301 static void scx_task_iter_start(struct scx_task_iter *iter) 1302 { 1303 BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & 1304 ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); 1305 1306 spin_lock_irq(&scx_tasks_lock); 1307 1308 iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; 1309 list_add(&iter->cursor.tasks_node, &scx_tasks); 1310 iter->locked = NULL; 1311 iter->cnt = 0; 1312 } 1313 1314 static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter) 1315 { 1316 if (iter->locked) { 1317 task_rq_unlock(iter->rq, iter->locked, &iter->rf); 1318 iter->locked = NULL; 1319 } 1320 } 1321 1322 /** 1323 * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator 1324 * @iter: iterator to unlock 1325 * 1326 * If @iter is in the middle of a locked iteration, it may be locking the rq of 1327 * the task currently being visited in addition to scx_tasks_lock. Unlock both. 1328 * This function can be safely called anytime during an iteration. 1329 */ 1330 static void scx_task_iter_unlock(struct scx_task_iter *iter) 1331 { 1332 __scx_task_iter_rq_unlock(iter); 1333 spin_unlock_irq(&scx_tasks_lock); 1334 } 1335 1336 /** 1337 * scx_task_iter_relock - Lock scx_tasks_lock released by scx_task_iter_unlock() 1338 * @iter: iterator to re-lock 1339 * 1340 * Re-lock scx_tasks_lock unlocked by scx_task_iter_unlock(). Note that it 1341 * doesn't re-lock the rq lock. Must be called before other iterator operations. 1342 */ 1343 static void scx_task_iter_relock(struct scx_task_iter *iter) 1344 { 1345 spin_lock_irq(&scx_tasks_lock); 1346 } 1347 1348 /** 1349 * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock 1350 * @iter: iterator to exit 1351 * 1352 * Exit a previously initialized @iter. Must be called with scx_tasks_lock held 1353 * which is released on return. If the iterator holds a task's rq lock, that rq 1354 * lock is also released. See scx_task_iter_start() for details. 1355 */ 1356 static void scx_task_iter_stop(struct scx_task_iter *iter) 1357 { 1358 list_del_init(&iter->cursor.tasks_node); 1359 scx_task_iter_unlock(iter); 1360 } 1361 1362 /** 1363 * scx_task_iter_next - Next task 1364 * @iter: iterator to walk 1365 * 1366 * Visit the next task. See scx_task_iter_start() for details. Locks are dropped 1367 * and re-acquired every %SCX_OPS_TASK_ITER_BATCH iterations to avoid causing 1368 * stalls by holding scx_tasks_lock for too long. 1369 */ 1370 static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter) 1371 { 1372 struct list_head *cursor = &iter->cursor.tasks_node; 1373 struct sched_ext_entity *pos; 1374 1375 if (!(++iter->cnt % SCX_OPS_TASK_ITER_BATCH)) { 1376 scx_task_iter_unlock(iter); 1377 cond_resched(); 1378 scx_task_iter_relock(iter); 1379 } 1380 1381 list_for_each_entry(pos, cursor, tasks_node) { 1382 if (&pos->tasks_node == &scx_tasks) 1383 return NULL; 1384 if (!(pos->flags & SCX_TASK_CURSOR)) { 1385 list_move(cursor, &pos->tasks_node); 1386 return container_of(pos, struct task_struct, scx); 1387 } 1388 } 1389 1390 /* can't happen, should always terminate at scx_tasks above */ 1391 BUG(); 1392 } 1393 1394 /** 1395 * scx_task_iter_next_locked - Next non-idle task with its rq locked 1396 * @iter: iterator to walk 1397 * @include_dead: Whether we should include dead tasks in the iteration 1398 * 1399 * Visit the non-idle task with its rq lock held. Allows callers to specify 1400 * whether they would like to filter out dead tasks. See scx_task_iter_start() 1401 * for details. 1402 */ 1403 static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter) 1404 { 1405 struct task_struct *p; 1406 1407 __scx_task_iter_rq_unlock(iter); 1408 1409 while ((p = scx_task_iter_next(iter))) { 1410 /* 1411 * scx_task_iter is used to prepare and move tasks into SCX 1412 * while loading the BPF scheduler and vice-versa while 1413 * unloading. The init_tasks ("swappers") should be excluded 1414 * from the iteration because: 1415 * 1416 * - It's unsafe to use __setschduler_prio() on an init_task to 1417 * determine the sched_class to use as it won't preserve its 1418 * idle_sched_class. 1419 * 1420 * - ops.init/exit_task() can easily be confused if called with 1421 * init_tasks as they, e.g., share PID 0. 1422 * 1423 * As init_tasks are never scheduled through SCX, they can be 1424 * skipped safely. Note that is_idle_task() which tests %PF_IDLE 1425 * doesn't work here: 1426 * 1427 * - %PF_IDLE may not be set for an init_task whose CPU hasn't 1428 * yet been onlined. 1429 * 1430 * - %PF_IDLE can be set on tasks that are not init_tasks. See 1431 * play_idle_precise() used by CONFIG_IDLE_INJECT. 1432 * 1433 * Test for idle_sched_class as only init_tasks are on it. 1434 */ 1435 if (p->sched_class != &idle_sched_class) 1436 break; 1437 } 1438 if (!p) 1439 return NULL; 1440 1441 iter->rq = task_rq_lock(p, &iter->rf); 1442 iter->locked = p; 1443 1444 return p; 1445 } 1446 1447 static enum scx_ops_enable_state scx_ops_enable_state(void) 1448 { 1449 return atomic_read(&scx_ops_enable_state_var); 1450 } 1451 1452 static enum scx_ops_enable_state 1453 scx_ops_set_enable_state(enum scx_ops_enable_state to) 1454 { 1455 return atomic_xchg(&scx_ops_enable_state_var, to); 1456 } 1457 1458 static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to, 1459 enum scx_ops_enable_state from) 1460 { 1461 int from_v = from; 1462 1463 return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to); 1464 } 1465 1466 static bool scx_rq_bypassing(struct rq *rq) 1467 { 1468 return unlikely(rq->scx.flags & SCX_RQ_BYPASSING); 1469 } 1470 1471 /** 1472 * wait_ops_state - Busy-wait the specified ops state to end 1473 * @p: target task 1474 * @opss: state to wait the end of 1475 * 1476 * Busy-wait for @p to transition out of @opss. This can only be used when the 1477 * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also 1478 * has load_acquire semantics to ensure that the caller can see the updates made 1479 * in the enqueueing and dispatching paths. 1480 */ 1481 static void wait_ops_state(struct task_struct *p, unsigned long opss) 1482 { 1483 do { 1484 cpu_relax(); 1485 } while (atomic_long_read_acquire(&p->scx.ops_state) == opss); 1486 } 1487 1488 /** 1489 * ops_cpu_valid - Verify a cpu number 1490 * @cpu: cpu number which came from a BPF ops 1491 * @where: extra information reported on error 1492 * 1493 * @cpu is a cpu number which came from the BPF scheduler and can be any value. 1494 * Verify that it is in range and one of the possible cpus. If invalid, trigger 1495 * an ops error. 1496 */ 1497 static bool ops_cpu_valid(s32 cpu, const char *where) 1498 { 1499 if (likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu))) { 1500 return true; 1501 } else { 1502 scx_ops_error("invalid CPU %d%s%s", cpu, 1503 where ? " " : "", where ?: ""); 1504 return false; 1505 } 1506 } 1507 1508 /** 1509 * ops_sanitize_err - Sanitize a -errno value 1510 * @ops_name: operation to blame on failure 1511 * @err: -errno value to sanitize 1512 * 1513 * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return 1514 * -%EPROTO. This is necessary because returning a rogue -errno up the chain can 1515 * cause misbehaviors. For an example, a large negative return from 1516 * ops.init_task() triggers an oops when passed up the call chain because the 1517 * value fails IS_ERR() test after being encoded with ERR_PTR() and then is 1518 * handled as a pointer. 1519 */ 1520 static int ops_sanitize_err(const char *ops_name, s32 err) 1521 { 1522 if (err < 0 && err >= -MAX_ERRNO) 1523 return err; 1524 1525 scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err); 1526 return -EPROTO; 1527 } 1528 1529 static void run_deferred(struct rq *rq) 1530 { 1531 process_ddsp_deferred_locals(rq); 1532 } 1533 1534 #ifdef CONFIG_SMP 1535 static void deferred_bal_cb_workfn(struct rq *rq) 1536 { 1537 run_deferred(rq); 1538 } 1539 #endif 1540 1541 static void deferred_irq_workfn(struct irq_work *irq_work) 1542 { 1543 struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work); 1544 1545 raw_spin_rq_lock(rq); 1546 run_deferred(rq); 1547 raw_spin_rq_unlock(rq); 1548 } 1549 1550 /** 1551 * schedule_deferred - Schedule execution of deferred actions on an rq 1552 * @rq: target rq 1553 * 1554 * Schedule execution of deferred actions on @rq. Must be called with @rq 1555 * locked. Deferred actions are executed with @rq locked but unpinned, and thus 1556 * can unlock @rq to e.g. migrate tasks to other rqs. 1557 */ 1558 static void schedule_deferred(struct rq *rq) 1559 { 1560 lockdep_assert_rq_held(rq); 1561 1562 #ifdef CONFIG_SMP 1563 /* 1564 * If in the middle of waking up a task, task_woken_scx() will be called 1565 * afterwards which will then run the deferred actions, no need to 1566 * schedule anything. 1567 */ 1568 if (rq->scx.flags & SCX_RQ_IN_WAKEUP) 1569 return; 1570 1571 /* 1572 * If in balance, the balance callbacks will be called before rq lock is 1573 * released. Schedule one. 1574 */ 1575 if (rq->scx.flags & SCX_RQ_IN_BALANCE) { 1576 queue_balance_callback(rq, &rq->scx.deferred_bal_cb, 1577 deferred_bal_cb_workfn); 1578 return; 1579 } 1580 #endif 1581 /* 1582 * No scheduler hooks available. Queue an irq work. They are executed on 1583 * IRQ re-enable which may take a bit longer than the scheduler hooks. 1584 * The above WAKEUP and BALANCE paths should cover most of the cases and 1585 * the time to IRQ re-enable shouldn't be long. 1586 */ 1587 irq_work_queue(&rq->scx.deferred_irq_work); 1588 } 1589 1590 /** 1591 * touch_core_sched - Update timestamp used for core-sched task ordering 1592 * @rq: rq to read clock from, must be locked 1593 * @p: task to update the timestamp for 1594 * 1595 * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to 1596 * implement global or local-DSQ FIFO ordering for core-sched. Should be called 1597 * when a task becomes runnable and its turn on the CPU ends (e.g. slice 1598 * exhaustion). 1599 */ 1600 static void touch_core_sched(struct rq *rq, struct task_struct *p) 1601 { 1602 lockdep_assert_rq_held(rq); 1603 1604 #ifdef CONFIG_SCHED_CORE 1605 /* 1606 * It's okay to update the timestamp spuriously. Use 1607 * sched_core_disabled() which is cheaper than enabled(). 1608 * 1609 * As this is used to determine ordering between tasks of sibling CPUs, 1610 * it may be better to use per-core dispatch sequence instead. 1611 */ 1612 if (!sched_core_disabled()) 1613 p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); 1614 #endif 1615 } 1616 1617 /** 1618 * touch_core_sched_dispatch - Update core-sched timestamp on dispatch 1619 * @rq: rq to read clock from, must be locked 1620 * @p: task being dispatched 1621 * 1622 * If the BPF scheduler implements custom core-sched ordering via 1623 * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO 1624 * ordering within each local DSQ. This function is called from dispatch paths 1625 * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. 1626 */ 1627 static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) 1628 { 1629 lockdep_assert_rq_held(rq); 1630 1631 #ifdef CONFIG_SCHED_CORE 1632 if (SCX_HAS_OP(core_sched_before)) 1633 touch_core_sched(rq, p); 1634 #endif 1635 } 1636 1637 static void update_curr_scx(struct rq *rq) 1638 { 1639 struct task_struct *curr = rq->curr; 1640 s64 delta_exec; 1641 1642 delta_exec = update_curr_common(rq); 1643 if (unlikely(delta_exec <= 0)) 1644 return; 1645 1646 if (curr->scx.slice != SCX_SLICE_INF) { 1647 curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); 1648 if (!curr->scx.slice) 1649 touch_core_sched(rq, curr); 1650 } 1651 } 1652 1653 static bool scx_dsq_priq_less(struct rb_node *node_a, 1654 const struct rb_node *node_b) 1655 { 1656 const struct task_struct *a = 1657 container_of(node_a, struct task_struct, scx.dsq_priq); 1658 const struct task_struct *b = 1659 container_of(node_b, struct task_struct, scx.dsq_priq); 1660 1661 return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime); 1662 } 1663 1664 static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta) 1665 { 1666 /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */ 1667 WRITE_ONCE(dsq->nr, dsq->nr + delta); 1668 } 1669 1670 static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p, 1671 u64 enq_flags) 1672 { 1673 bool is_local = dsq->id == SCX_DSQ_LOCAL; 1674 1675 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); 1676 WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || 1677 !RB_EMPTY_NODE(&p->scx.dsq_priq)); 1678 1679 if (!is_local) { 1680 raw_spin_lock(&dsq->lock); 1681 if (unlikely(dsq->id == SCX_DSQ_INVALID)) { 1682 scx_ops_error("attempting to dispatch to a destroyed dsq"); 1683 /* fall back to the global dsq */ 1684 raw_spin_unlock(&dsq->lock); 1685 dsq = find_global_dsq(p); 1686 raw_spin_lock(&dsq->lock); 1687 } 1688 } 1689 1690 if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) && 1691 (enq_flags & SCX_ENQ_DSQ_PRIQ))) { 1692 /* 1693 * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from 1694 * their FIFO queues. To avoid confusion and accidentally 1695 * starving vtime-dispatched tasks by FIFO-dispatched tasks, we 1696 * disallow any internal DSQ from doing vtime ordering of 1697 * tasks. 1698 */ 1699 scx_ops_error("cannot use vtime ordering for built-in DSQs"); 1700 enq_flags &= ~SCX_ENQ_DSQ_PRIQ; 1701 } 1702 1703 if (enq_flags & SCX_ENQ_DSQ_PRIQ) { 1704 struct rb_node *rbp; 1705 1706 /* 1707 * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are 1708 * linked to both the rbtree and list on PRIQs, this can only be 1709 * tested easily when adding the first task. 1710 */ 1711 if (unlikely(RB_EMPTY_ROOT(&dsq->priq) && 1712 nldsq_next_task(dsq, NULL, false))) 1713 scx_ops_error("DSQ ID 0x%016llx already had FIFO-enqueued tasks", 1714 dsq->id); 1715 1716 p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ; 1717 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less); 1718 1719 /* 1720 * Find the previous task and insert after it on the list so 1721 * that @dsq->list is vtime ordered. 1722 */ 1723 rbp = rb_prev(&p->scx.dsq_priq); 1724 if (rbp) { 1725 struct task_struct *prev = 1726 container_of(rbp, struct task_struct, 1727 scx.dsq_priq); 1728 list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node); 1729 } else { 1730 list_add(&p->scx.dsq_list.node, &dsq->list); 1731 } 1732 } else { 1733 /* a FIFO DSQ shouldn't be using PRIQ enqueuing */ 1734 if (unlikely(!RB_EMPTY_ROOT(&dsq->priq))) 1735 scx_ops_error("DSQ ID 0x%016llx already had PRIQ-enqueued tasks", 1736 dsq->id); 1737 1738 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) 1739 list_add(&p->scx.dsq_list.node, &dsq->list); 1740 else 1741 list_add_tail(&p->scx.dsq_list.node, &dsq->list); 1742 } 1743 1744 /* seq records the order tasks are queued, used by BPF DSQ iterator */ 1745 dsq->seq++; 1746 p->scx.dsq_seq = dsq->seq; 1747 1748 dsq_mod_nr(dsq, 1); 1749 p->scx.dsq = dsq; 1750 1751 /* 1752 * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the 1753 * direct dispatch path, but we clear them here because the direct 1754 * dispatch verdict may be overridden on the enqueue path during e.g. 1755 * bypass. 1756 */ 1757 p->scx.ddsp_dsq_id = SCX_DSQ_INVALID; 1758 p->scx.ddsp_enq_flags = 0; 1759 1760 /* 1761 * We're transitioning out of QUEUEING or DISPATCHING. store_release to 1762 * match waiters' load_acquire. 1763 */ 1764 if (enq_flags & SCX_ENQ_CLEAR_OPSS) 1765 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 1766 1767 if (is_local) { 1768 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq); 1769 bool preempt = false; 1770 1771 if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && 1772 rq->curr->sched_class == &ext_sched_class) { 1773 rq->curr->scx.slice = 0; 1774 preempt = true; 1775 } 1776 1777 if (preempt || sched_class_above(&ext_sched_class, 1778 rq->curr->sched_class)) 1779 resched_curr(rq); 1780 } else { 1781 raw_spin_unlock(&dsq->lock); 1782 } 1783 } 1784 1785 static void task_unlink_from_dsq(struct task_struct *p, 1786 struct scx_dispatch_q *dsq) 1787 { 1788 WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node)); 1789 1790 if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) { 1791 rb_erase(&p->scx.dsq_priq, &dsq->priq); 1792 RB_CLEAR_NODE(&p->scx.dsq_priq); 1793 p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; 1794 } 1795 1796 list_del_init(&p->scx.dsq_list.node); 1797 dsq_mod_nr(dsq, -1); 1798 } 1799 1800 static void dispatch_dequeue(struct rq *rq, struct task_struct *p) 1801 { 1802 struct scx_dispatch_q *dsq = p->scx.dsq; 1803 bool is_local = dsq == &rq->scx.local_dsq; 1804 1805 if (!dsq) { 1806 /* 1807 * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals. 1808 * Unlinking is all that's needed to cancel. 1809 */ 1810 if (unlikely(!list_empty(&p->scx.dsq_list.node))) 1811 list_del_init(&p->scx.dsq_list.node); 1812 1813 /* 1814 * When dispatching directly from the BPF scheduler to a local 1815 * DSQ, the task isn't associated with any DSQ but 1816 * @p->scx.holding_cpu may be set under the protection of 1817 * %SCX_OPSS_DISPATCHING. 1818 */ 1819 if (p->scx.holding_cpu >= 0) 1820 p->scx.holding_cpu = -1; 1821 1822 return; 1823 } 1824 1825 if (!is_local) 1826 raw_spin_lock(&dsq->lock); 1827 1828 /* 1829 * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't 1830 * change underneath us. 1831 */ 1832 if (p->scx.holding_cpu < 0) { 1833 /* @p must still be on @dsq, dequeue */ 1834 task_unlink_from_dsq(p, dsq); 1835 } else { 1836 /* 1837 * We're racing against dispatch_to_local_dsq() which already 1838 * removed @p from @dsq and set @p->scx.holding_cpu. Clear the 1839 * holding_cpu which tells dispatch_to_local_dsq() that it lost 1840 * the race. 1841 */ 1842 WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); 1843 p->scx.holding_cpu = -1; 1844 } 1845 p->scx.dsq = NULL; 1846 1847 if (!is_local) 1848 raw_spin_unlock(&dsq->lock); 1849 } 1850 1851 static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id, 1852 struct task_struct *p) 1853 { 1854 struct scx_dispatch_q *dsq; 1855 1856 if (dsq_id == SCX_DSQ_LOCAL) 1857 return &rq->scx.local_dsq; 1858 1859 if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { 1860 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; 1861 1862 if (!ops_cpu_valid(cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict")) 1863 return find_global_dsq(p); 1864 1865 return &cpu_rq(cpu)->scx.local_dsq; 1866 } 1867 1868 if (dsq_id == SCX_DSQ_GLOBAL) 1869 dsq = find_global_dsq(p); 1870 else 1871 dsq = find_user_dsq(dsq_id); 1872 1873 if (unlikely(!dsq)) { 1874 scx_ops_error("non-existent DSQ 0x%llx for %s[%d]", 1875 dsq_id, p->comm, p->pid); 1876 return find_global_dsq(p); 1877 } 1878 1879 return dsq; 1880 } 1881 1882 static void mark_direct_dispatch(struct task_struct *ddsp_task, 1883 struct task_struct *p, u64 dsq_id, 1884 u64 enq_flags) 1885 { 1886 /* 1887 * Mark that dispatch already happened from ops.select_cpu() or 1888 * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value 1889 * which can never match a valid task pointer. 1890 */ 1891 __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH)); 1892 1893 /* @p must match the task on the enqueue path */ 1894 if (unlikely(p != ddsp_task)) { 1895 if (IS_ERR(ddsp_task)) 1896 scx_ops_error("%s[%d] already direct-dispatched", 1897 p->comm, p->pid); 1898 else 1899 scx_ops_error("scheduling for %s[%d] but trying to direct-dispatch %s[%d]", 1900 ddsp_task->comm, ddsp_task->pid, 1901 p->comm, p->pid); 1902 return; 1903 } 1904 1905 WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID); 1906 WARN_ON_ONCE(p->scx.ddsp_enq_flags); 1907 1908 p->scx.ddsp_dsq_id = dsq_id; 1909 p->scx.ddsp_enq_flags = enq_flags; 1910 } 1911 1912 static void direct_dispatch(struct task_struct *p, u64 enq_flags) 1913 { 1914 struct rq *rq = task_rq(p); 1915 struct scx_dispatch_q *dsq = 1916 find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p); 1917 1918 touch_core_sched_dispatch(rq, p); 1919 1920 p->scx.ddsp_enq_flags |= enq_flags; 1921 1922 /* 1923 * We are in the enqueue path with @rq locked and pinned, and thus can't 1924 * double lock a remote rq and enqueue to its local DSQ. For 1925 * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer 1926 * the enqueue so that it's executed when @rq can be unlocked. 1927 */ 1928 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) { 1929 unsigned long opss; 1930 1931 opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK; 1932 1933 switch (opss & SCX_OPSS_STATE_MASK) { 1934 case SCX_OPSS_NONE: 1935 break; 1936 case SCX_OPSS_QUEUEING: 1937 /* 1938 * As @p was never passed to the BPF side, _release is 1939 * not strictly necessary. Still do it for consistency. 1940 */ 1941 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 1942 break; 1943 default: 1944 WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()", 1945 p->comm, p->pid, opss); 1946 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 1947 break; 1948 } 1949 1950 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); 1951 list_add_tail(&p->scx.dsq_list.node, 1952 &rq->scx.ddsp_deferred_locals); 1953 schedule_deferred(rq); 1954 return; 1955 } 1956 1957 dispatch_enqueue(dsq, p, p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS); 1958 } 1959 1960 static bool scx_rq_online(struct rq *rq) 1961 { 1962 /* 1963 * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates 1964 * the online state as seen from the BPF scheduler. cpu_active() test 1965 * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will 1966 * stay set until the current scheduling operation is complete even if 1967 * we aren't locking @rq. 1968 */ 1969 return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); 1970 } 1971 1972 static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, 1973 int sticky_cpu) 1974 { 1975 struct task_struct **ddsp_taskp; 1976 unsigned long qseq; 1977 1978 WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED)); 1979 1980 /* rq migration */ 1981 if (sticky_cpu == cpu_of(rq)) 1982 goto local_norefill; 1983 1984 /* 1985 * If !scx_rq_online(), we already told the BPF scheduler that the CPU 1986 * is offline and are just running the hotplug path. Don't bother the 1987 * BPF scheduler. 1988 */ 1989 if (!scx_rq_online(rq)) 1990 goto local; 1991 1992 if (scx_rq_bypassing(rq)) 1993 goto global; 1994 1995 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) 1996 goto direct; 1997 1998 /* see %SCX_OPS_ENQ_EXITING */ 1999 if (!static_branch_unlikely(&scx_ops_enq_exiting) && 2000 unlikely(p->flags & PF_EXITING)) 2001 goto local; 2002 2003 if (!SCX_HAS_OP(enqueue)) 2004 goto global; 2005 2006 /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ 2007 qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; 2008 2009 WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); 2010 atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); 2011 2012 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); 2013 WARN_ON_ONCE(*ddsp_taskp); 2014 *ddsp_taskp = p; 2015 2016 SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags); 2017 2018 *ddsp_taskp = NULL; 2019 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) 2020 goto direct; 2021 2022 /* 2023 * If not directly dispatched, QUEUEING isn't clear yet and dispatch or 2024 * dequeue may be waiting. The store_release matches their load_acquire. 2025 */ 2026 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); 2027 return; 2028 2029 direct: 2030 direct_dispatch(p, enq_flags); 2031 return; 2032 2033 local: 2034 /* 2035 * For task-ordering, slice refill must be treated as implying the end 2036 * of the current slice. Otherwise, the longer @p stays on the CPU, the 2037 * higher priority it becomes from scx_prio_less()'s POV. 2038 */ 2039 touch_core_sched(rq, p); 2040 p->scx.slice = SCX_SLICE_DFL; 2041 local_norefill: 2042 dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags); 2043 return; 2044 2045 global: 2046 touch_core_sched(rq, p); /* see the comment in local: */ 2047 p->scx.slice = SCX_SLICE_DFL; 2048 dispatch_enqueue(find_global_dsq(p), p, enq_flags); 2049 } 2050 2051 static bool task_runnable(const struct task_struct *p) 2052 { 2053 return !list_empty(&p->scx.runnable_node); 2054 } 2055 2056 static void set_task_runnable(struct rq *rq, struct task_struct *p) 2057 { 2058 lockdep_assert_rq_held(rq); 2059 2060 if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) { 2061 p->scx.runnable_at = jiffies; 2062 p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT; 2063 } 2064 2065 /* 2066 * list_add_tail() must be used. scx_ops_bypass() depends on tasks being 2067 * appened to the runnable_list. 2068 */ 2069 list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list); 2070 } 2071 2072 static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) 2073 { 2074 list_del_init(&p->scx.runnable_node); 2075 if (reset_runnable_at) 2076 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 2077 } 2078 2079 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) 2080 { 2081 int sticky_cpu = p->scx.sticky_cpu; 2082 2083 if (enq_flags & ENQUEUE_WAKEUP) 2084 rq->scx.flags |= SCX_RQ_IN_WAKEUP; 2085 2086 enq_flags |= rq->scx.extra_enq_flags; 2087 2088 if (sticky_cpu >= 0) 2089 p->scx.sticky_cpu = -1; 2090 2091 /* 2092 * Restoring a running task will be immediately followed by 2093 * set_next_task_scx() which expects the task to not be on the BPF 2094 * scheduler as tasks can only start running through local DSQs. Force 2095 * direct-dispatch into the local DSQ by setting the sticky_cpu. 2096 */ 2097 if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) 2098 sticky_cpu = cpu_of(rq); 2099 2100 if (p->scx.flags & SCX_TASK_QUEUED) { 2101 WARN_ON_ONCE(!task_runnable(p)); 2102 goto out; 2103 } 2104 2105 set_task_runnable(rq, p); 2106 p->scx.flags |= SCX_TASK_QUEUED; 2107 rq->scx.nr_running++; 2108 add_nr_running(rq, 1); 2109 2110 if (SCX_HAS_OP(runnable) && !task_on_rq_migrating(p)) 2111 SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); 2112 2113 if (enq_flags & SCX_ENQ_WAKEUP) 2114 touch_core_sched(rq, p); 2115 2116 do_enqueue_task(rq, p, enq_flags, sticky_cpu); 2117 out: 2118 rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; 2119 } 2120 2121 static void ops_dequeue(struct task_struct *p, u64 deq_flags) 2122 { 2123 unsigned long opss; 2124 2125 /* dequeue is always temporary, don't reset runnable_at */ 2126 clr_task_runnable(p, false); 2127 2128 /* acquire ensures that we see the preceding updates on QUEUED */ 2129 opss = atomic_long_read_acquire(&p->scx.ops_state); 2130 2131 switch (opss & SCX_OPSS_STATE_MASK) { 2132 case SCX_OPSS_NONE: 2133 break; 2134 case SCX_OPSS_QUEUEING: 2135 /* 2136 * QUEUEING is started and finished while holding @p's rq lock. 2137 * As we're holding the rq lock now, we shouldn't see QUEUEING. 2138 */ 2139 BUG(); 2140 case SCX_OPSS_QUEUED: 2141 if (SCX_HAS_OP(dequeue)) 2142 SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags); 2143 2144 if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, 2145 SCX_OPSS_NONE)) 2146 break; 2147 fallthrough; 2148 case SCX_OPSS_DISPATCHING: 2149 /* 2150 * If @p is being dispatched from the BPF scheduler to a DSQ, 2151 * wait for the transfer to complete so that @p doesn't get 2152 * added to its DSQ after dequeueing is complete. 2153 * 2154 * As we're waiting on DISPATCHING with the rq locked, the 2155 * dispatching side shouldn't try to lock the rq while 2156 * DISPATCHING is set. See dispatch_to_local_dsq(). 2157 * 2158 * DISPATCHING shouldn't have qseq set and control can reach 2159 * here with NONE @opss from the above QUEUED case block. 2160 * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss. 2161 */ 2162 wait_ops_state(p, SCX_OPSS_DISPATCHING); 2163 BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); 2164 break; 2165 } 2166 } 2167 2168 static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) 2169 { 2170 if (!(p->scx.flags & SCX_TASK_QUEUED)) { 2171 WARN_ON_ONCE(task_runnable(p)); 2172 return true; 2173 } 2174 2175 ops_dequeue(p, deq_flags); 2176 2177 /* 2178 * A currently running task which is going off @rq first gets dequeued 2179 * and then stops running. As we want running <-> stopping transitions 2180 * to be contained within runnable <-> quiescent transitions, trigger 2181 * ->stopping() early here instead of in put_prev_task_scx(). 2182 * 2183 * @p may go through multiple stopping <-> running transitions between 2184 * here and put_prev_task_scx() if task attribute changes occur while 2185 * balance_scx() leaves @rq unlocked. However, they don't contain any 2186 * information meaningful to the BPF scheduler and can be suppressed by 2187 * skipping the callbacks if the task is !QUEUED. 2188 */ 2189 if (SCX_HAS_OP(stopping) && task_current(rq, p)) { 2190 update_curr_scx(rq); 2191 SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); 2192 } 2193 2194 if (SCX_HAS_OP(quiescent) && !task_on_rq_migrating(p)) 2195 SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); 2196 2197 if (deq_flags & SCX_DEQ_SLEEP) 2198 p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; 2199 else 2200 p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP; 2201 2202 p->scx.flags &= ~SCX_TASK_QUEUED; 2203 rq->scx.nr_running--; 2204 sub_nr_running(rq, 1); 2205 2206 dispatch_dequeue(rq, p); 2207 return true; 2208 } 2209 2210 static void yield_task_scx(struct rq *rq) 2211 { 2212 struct task_struct *p = rq->curr; 2213 2214 if (SCX_HAS_OP(yield)) 2215 SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL); 2216 else 2217 p->scx.slice = 0; 2218 } 2219 2220 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to) 2221 { 2222 struct task_struct *from = rq->curr; 2223 2224 if (SCX_HAS_OP(yield)) 2225 return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to); 2226 else 2227 return false; 2228 } 2229 2230 static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags, 2231 struct scx_dispatch_q *src_dsq, 2232 struct rq *dst_rq) 2233 { 2234 struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq; 2235 2236 /* @dsq is locked and @p is on @dst_rq */ 2237 lockdep_assert_held(&src_dsq->lock); 2238 lockdep_assert_rq_held(dst_rq); 2239 2240 WARN_ON_ONCE(p->scx.holding_cpu >= 0); 2241 2242 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) 2243 list_add(&p->scx.dsq_list.node, &dst_dsq->list); 2244 else 2245 list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); 2246 2247 dsq_mod_nr(dst_dsq, 1); 2248 p->scx.dsq = dst_dsq; 2249 } 2250 2251 #ifdef CONFIG_SMP 2252 /** 2253 * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ 2254 * @p: task to move 2255 * @enq_flags: %SCX_ENQ_* 2256 * @src_rq: rq to move the task from, locked on entry, released on return 2257 * @dst_rq: rq to move the task into, locked on return 2258 * 2259 * Move @p which is currently on @src_rq to @dst_rq's local DSQ. 2260 */ 2261 static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, 2262 struct rq *src_rq, struct rq *dst_rq) 2263 { 2264 lockdep_assert_rq_held(src_rq); 2265 2266 /* the following marks @p MIGRATING which excludes dequeue */ 2267 deactivate_task(src_rq, p, 0); 2268 set_task_cpu(p, cpu_of(dst_rq)); 2269 p->scx.sticky_cpu = cpu_of(dst_rq); 2270 2271 raw_spin_rq_unlock(src_rq); 2272 raw_spin_rq_lock(dst_rq); 2273 2274 /* 2275 * We want to pass scx-specific enq_flags but activate_task() will 2276 * truncate the upper 32 bit. As we own @rq, we can pass them through 2277 * @rq->scx.extra_enq_flags instead. 2278 */ 2279 WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); 2280 WARN_ON_ONCE(dst_rq->scx.extra_enq_flags); 2281 dst_rq->scx.extra_enq_flags = enq_flags; 2282 activate_task(dst_rq, p, 0); 2283 dst_rq->scx.extra_enq_flags = 0; 2284 } 2285 2286 /* 2287 * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two 2288 * differences: 2289 * 2290 * - is_cpu_allowed() asks "Can this task run on this CPU?" while 2291 * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to 2292 * this CPU?". 2293 * 2294 * While migration is disabled, is_cpu_allowed() has to say "yes" as the task 2295 * must be allowed to finish on the CPU that it's currently on regardless of 2296 * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the 2297 * BPF scheduler shouldn't attempt to migrate a task which has migration 2298 * disabled. 2299 * 2300 * - The BPF scheduler is bypassed while the rq is offline and we can always say 2301 * no to the BPF scheduler initiated migrations while offline. 2302 */ 2303 static bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq, 2304 bool trigger_error) 2305 { 2306 int cpu = cpu_of(rq); 2307 2308 /* 2309 * We don't require the BPF scheduler to avoid dispatching to offline 2310 * CPUs mostly for convenience but also because CPUs can go offline 2311 * between scx_bpf_dispatch() calls and here. Trigger error iff the 2312 * picked CPU is outside the allowed mask. 2313 */ 2314 if (!task_allowed_on_cpu(p, cpu)) { 2315 if (trigger_error) 2316 scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]", 2317 cpu_of(rq), p->comm, p->pid); 2318 return false; 2319 } 2320 2321 if (unlikely(is_migration_disabled(p))) 2322 return false; 2323 2324 if (!scx_rq_online(rq)) 2325 return false; 2326 2327 return true; 2328 } 2329 2330 /** 2331 * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq 2332 * @p: target task 2333 * @dsq: locked DSQ @p is currently on 2334 * @src_rq: rq @p is currently on, stable with @dsq locked 2335 * 2336 * Called with @dsq locked but no rq's locked. We want to move @p to a different 2337 * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is 2338 * required when transferring into a local DSQ. Even when transferring into a 2339 * non-local DSQ, it's better to use the same mechanism to protect against 2340 * dequeues and maintain the invariant that @p->scx.dsq can only change while 2341 * @src_rq is locked, which e.g. scx_dump_task() depends on. 2342 * 2343 * We want to grab @src_rq but that can deadlock if we try while locking @dsq, 2344 * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As 2345 * this may race with dequeue, which can't drop the rq lock or fail, do a little 2346 * dancing from our side. 2347 * 2348 * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets 2349 * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu 2350 * would be cleared to -1. While other cpus may have updated it to different 2351 * values afterwards, as this operation can't be preempted or recurse, the 2352 * holding_cpu can never become this CPU again before we're done. Thus, we can 2353 * tell whether we lost to dequeue by testing whether the holding_cpu still 2354 * points to this CPU. See dispatch_dequeue() for the counterpart. 2355 * 2356 * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is 2357 * still valid. %false if lost to dequeue. 2358 */ 2359 static bool unlink_dsq_and_lock_src_rq(struct task_struct *p, 2360 struct scx_dispatch_q *dsq, 2361 struct rq *src_rq) 2362 { 2363 s32 cpu = raw_smp_processor_id(); 2364 2365 lockdep_assert_held(&dsq->lock); 2366 2367 WARN_ON_ONCE(p->scx.holding_cpu >= 0); 2368 task_unlink_from_dsq(p, dsq); 2369 p->scx.holding_cpu = cpu; 2370 2371 raw_spin_unlock(&dsq->lock); 2372 raw_spin_rq_lock(src_rq); 2373 2374 /* task_rq couldn't have changed if we're still the holding cpu */ 2375 return likely(p->scx.holding_cpu == cpu) && 2376 !WARN_ON_ONCE(src_rq != task_rq(p)); 2377 } 2378 2379 static bool consume_remote_task(struct rq *this_rq, struct task_struct *p, 2380 struct scx_dispatch_q *dsq, struct rq *src_rq) 2381 { 2382 raw_spin_rq_unlock(this_rq); 2383 2384 if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) { 2385 move_remote_task_to_local_dsq(p, 0, src_rq, this_rq); 2386 return true; 2387 } else { 2388 raw_spin_rq_unlock(src_rq); 2389 raw_spin_rq_lock(this_rq); 2390 return false; 2391 } 2392 } 2393 #else /* CONFIG_SMP */ 2394 static inline void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { WARN_ON_ONCE(1); } 2395 static inline bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq, bool trigger_error) { return false; } 2396 static inline bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq) { return false; } 2397 #endif /* CONFIG_SMP */ 2398 2399 static bool consume_dispatch_q(struct rq *rq, struct scx_dispatch_q *dsq) 2400 { 2401 struct task_struct *p; 2402 retry: 2403 /* 2404 * The caller can't expect to successfully consume a task if the task's 2405 * addition to @dsq isn't guaranteed to be visible somehow. Test 2406 * @dsq->list without locking and skip if it seems empty. 2407 */ 2408 if (list_empty(&dsq->list)) 2409 return false; 2410 2411 raw_spin_lock(&dsq->lock); 2412 2413 nldsq_for_each_task(p, dsq) { 2414 struct rq *task_rq = task_rq(p); 2415 2416 if (rq == task_rq) { 2417 task_unlink_from_dsq(p, dsq); 2418 move_local_task_to_local_dsq(p, 0, dsq, rq); 2419 raw_spin_unlock(&dsq->lock); 2420 return true; 2421 } 2422 2423 if (task_can_run_on_remote_rq(p, rq, false)) { 2424 if (likely(consume_remote_task(rq, p, dsq, task_rq))) 2425 return true; 2426 goto retry; 2427 } 2428 } 2429 2430 raw_spin_unlock(&dsq->lock); 2431 return false; 2432 } 2433 2434 static bool consume_global_dsq(struct rq *rq) 2435 { 2436 int node = cpu_to_node(cpu_of(rq)); 2437 2438 return consume_dispatch_q(rq, global_dsqs[node]); 2439 } 2440 2441 /** 2442 * dispatch_to_local_dsq - Dispatch a task to a local dsq 2443 * @rq: current rq which is locked 2444 * @dst_dsq: destination DSQ 2445 * @p: task to dispatch 2446 * @enq_flags: %SCX_ENQ_* 2447 * 2448 * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local 2449 * DSQ. This function performs all the synchronization dancing needed because 2450 * local DSQs are protected with rq locks. 2451 * 2452 * The caller must have exclusive ownership of @p (e.g. through 2453 * %SCX_OPSS_DISPATCHING). 2454 */ 2455 static void dispatch_to_local_dsq(struct rq *rq, struct scx_dispatch_q *dst_dsq, 2456 struct task_struct *p, u64 enq_flags) 2457 { 2458 struct rq *src_rq = task_rq(p); 2459 struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); 2460 2461 /* 2462 * We're synchronized against dequeue through DISPATCHING. As @p can't 2463 * be dequeued, its task_rq and cpus_allowed are stable too. 2464 * 2465 * If dispatching to @rq that @p is already on, no lock dancing needed. 2466 */ 2467 if (rq == src_rq && rq == dst_rq) { 2468 dispatch_enqueue(dst_dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); 2469 return; 2470 } 2471 2472 #ifdef CONFIG_SMP 2473 if (unlikely(!task_can_run_on_remote_rq(p, dst_rq, true))) { 2474 dispatch_enqueue(find_global_dsq(p), p, 2475 enq_flags | SCX_ENQ_CLEAR_OPSS); 2476 return; 2477 } 2478 2479 /* 2480 * @p is on a possibly remote @src_rq which we need to lock to move the 2481 * task. If dequeue is in progress, it'd be locking @src_rq and waiting 2482 * on DISPATCHING, so we can't grab @src_rq lock while holding 2483 * DISPATCHING. 2484 * 2485 * As DISPATCHING guarantees that @p is wholly ours, we can pretend that 2486 * we're moving from a DSQ and use the same mechanism - mark the task 2487 * under transfer with holding_cpu, release DISPATCHING and then follow 2488 * the same protocol. See unlink_dsq_and_lock_src_rq(). 2489 */ 2490 p->scx.holding_cpu = raw_smp_processor_id(); 2491 2492 /* store_release ensures that dequeue sees the above */ 2493 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE); 2494 2495 /* switch to @src_rq lock */ 2496 if (rq != src_rq) { 2497 raw_spin_rq_unlock(rq); 2498 raw_spin_rq_lock(src_rq); 2499 } 2500 2501 /* task_rq couldn't have changed if we're still the holding cpu */ 2502 if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && 2503 !WARN_ON_ONCE(src_rq != task_rq(p))) { 2504 /* 2505 * If @p is staying on the same rq, there's no need to go 2506 * through the full deactivate/activate cycle. Optimize by 2507 * abbreviating move_remote_task_to_local_dsq(). 2508 */ 2509 if (src_rq == dst_rq) { 2510 p->scx.holding_cpu = -1; 2511 dispatch_enqueue(&dst_rq->scx.local_dsq, p, enq_flags); 2512 } else { 2513 move_remote_task_to_local_dsq(p, enq_flags, 2514 src_rq, dst_rq); 2515 } 2516 2517 /* if the destination CPU is idle, wake it up */ 2518 if (sched_class_above(p->sched_class, dst_rq->curr->sched_class)) 2519 resched_curr(dst_rq); 2520 } 2521 2522 /* switch back to @rq lock */ 2523 if (rq != dst_rq) { 2524 raw_spin_rq_unlock(dst_rq); 2525 raw_spin_rq_lock(rq); 2526 } 2527 #else /* CONFIG_SMP */ 2528 BUG(); /* control can not reach here on UP */ 2529 #endif /* CONFIG_SMP */ 2530 } 2531 2532 /** 2533 * finish_dispatch - Asynchronously finish dispatching a task 2534 * @rq: current rq which is locked 2535 * @p: task to finish dispatching 2536 * @qseq_at_dispatch: qseq when @p started getting dispatched 2537 * @dsq_id: destination DSQ ID 2538 * @enq_flags: %SCX_ENQ_* 2539 * 2540 * Dispatching to local DSQs may need to wait for queueing to complete or 2541 * require rq lock dancing. As we don't wanna do either while inside 2542 * ops.dispatch() to avoid locking order inversion, we split dispatching into 2543 * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the 2544 * task and its qseq. Once ops.dispatch() returns, this function is called to 2545 * finish up. 2546 * 2547 * There is no guarantee that @p is still valid for dispatching or even that it 2548 * was valid in the first place. Make sure that the task is still owned by the 2549 * BPF scheduler and claim the ownership before dispatching. 2550 */ 2551 static void finish_dispatch(struct rq *rq, struct task_struct *p, 2552 unsigned long qseq_at_dispatch, 2553 u64 dsq_id, u64 enq_flags) 2554 { 2555 struct scx_dispatch_q *dsq; 2556 unsigned long opss; 2557 2558 touch_core_sched_dispatch(rq, p); 2559 retry: 2560 /* 2561 * No need for _acquire here. @p is accessed only after a successful 2562 * try_cmpxchg to DISPATCHING. 2563 */ 2564 opss = atomic_long_read(&p->scx.ops_state); 2565 2566 switch (opss & SCX_OPSS_STATE_MASK) { 2567 case SCX_OPSS_DISPATCHING: 2568 case SCX_OPSS_NONE: 2569 /* someone else already got to it */ 2570 return; 2571 case SCX_OPSS_QUEUED: 2572 /* 2573 * If qseq doesn't match, @p has gone through at least one 2574 * dispatch/dequeue and re-enqueue cycle between 2575 * scx_bpf_dispatch() and here and we have no claim on it. 2576 */ 2577 if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch) 2578 return; 2579 2580 /* 2581 * While we know @p is accessible, we don't yet have a claim on 2582 * it - the BPF scheduler is allowed to dispatch tasks 2583 * spuriously and there can be a racing dequeue attempt. Let's 2584 * claim @p by atomically transitioning it from QUEUED to 2585 * DISPATCHING. 2586 */ 2587 if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss, 2588 SCX_OPSS_DISPATCHING))) 2589 break; 2590 goto retry; 2591 case SCX_OPSS_QUEUEING: 2592 /* 2593 * do_enqueue_task() is in the process of transferring the task 2594 * to the BPF scheduler while holding @p's rq lock. As we aren't 2595 * holding any kernel or BPF resource that the enqueue path may 2596 * depend upon, it's safe to wait. 2597 */ 2598 wait_ops_state(p, opss); 2599 goto retry; 2600 } 2601 2602 BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED)); 2603 2604 dsq = find_dsq_for_dispatch(this_rq(), dsq_id, p); 2605 2606 if (dsq->id == SCX_DSQ_LOCAL) 2607 dispatch_to_local_dsq(rq, dsq, p, enq_flags); 2608 else 2609 dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); 2610 } 2611 2612 static void flush_dispatch_buf(struct rq *rq) 2613 { 2614 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 2615 u32 u; 2616 2617 for (u = 0; u < dspc->cursor; u++) { 2618 struct scx_dsp_buf_ent *ent = &dspc->buf[u]; 2619 2620 finish_dispatch(rq, ent->task, ent->qseq, ent->dsq_id, 2621 ent->enq_flags); 2622 } 2623 2624 dspc->nr_tasks += dspc->cursor; 2625 dspc->cursor = 0; 2626 } 2627 2628 static int balance_one(struct rq *rq, struct task_struct *prev) 2629 { 2630 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 2631 bool prev_on_scx = prev->sched_class == &ext_sched_class; 2632 int nr_loops = SCX_DSP_MAX_LOOPS; 2633 2634 lockdep_assert_rq_held(rq); 2635 rq->scx.flags |= SCX_RQ_IN_BALANCE; 2636 rq->scx.flags &= ~SCX_RQ_BAL_KEEP; 2637 2638 if (static_branch_unlikely(&scx_ops_cpu_preempt) && 2639 unlikely(rq->scx.cpu_released)) { 2640 /* 2641 * If the previous sched_class for the current CPU was not SCX, 2642 * notify the BPF scheduler that it again has control of the 2643 * core. This callback complements ->cpu_release(), which is 2644 * emitted in scx_next_task_picked(). 2645 */ 2646 if (SCX_HAS_OP(cpu_acquire)) 2647 SCX_CALL_OP(0, cpu_acquire, cpu_of(rq), NULL); 2648 rq->scx.cpu_released = false; 2649 } 2650 2651 if (prev_on_scx) { 2652 update_curr_scx(rq); 2653 2654 /* 2655 * If @prev is runnable & has slice left, it has priority and 2656 * fetching more just increases latency for the fetched tasks. 2657 * Tell pick_task_scx() to keep running @prev. If the BPF 2658 * scheduler wants to handle this explicitly, it should 2659 * implement ->cpu_release(). 2660 * 2661 * See scx_ops_disable_workfn() for the explanation on the 2662 * bypassing test. 2663 */ 2664 if ((prev->scx.flags & SCX_TASK_QUEUED) && 2665 prev->scx.slice && !scx_rq_bypassing(rq)) { 2666 rq->scx.flags |= SCX_RQ_BAL_KEEP; 2667 goto has_tasks; 2668 } 2669 } 2670 2671 /* if there already are tasks to run, nothing to do */ 2672 if (rq->scx.local_dsq.nr) 2673 goto has_tasks; 2674 2675 if (consume_global_dsq(rq)) 2676 goto has_tasks; 2677 2678 if (!SCX_HAS_OP(dispatch) || scx_rq_bypassing(rq) || !scx_rq_online(rq)) 2679 goto no_tasks; 2680 2681 dspc->rq = rq; 2682 2683 /* 2684 * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock, 2685 * the local DSQ might still end up empty after a successful 2686 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch() 2687 * produced some tasks, retry. The BPF scheduler may depend on this 2688 * looping behavior to simplify its implementation. 2689 */ 2690 do { 2691 dspc->nr_tasks = 0; 2692 2693 SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq), 2694 prev_on_scx ? prev : NULL); 2695 2696 flush_dispatch_buf(rq); 2697 2698 if (rq->scx.local_dsq.nr) 2699 goto has_tasks; 2700 if (consume_global_dsq(rq)) 2701 goto has_tasks; 2702 2703 /* 2704 * ops.dispatch() can trap us in this loop by repeatedly 2705 * dispatching ineligible tasks. Break out once in a while to 2706 * allow the watchdog to run. As IRQ can't be enabled in 2707 * balance(), we want to complete this scheduling cycle and then 2708 * start a new one. IOW, we want to call resched_curr() on the 2709 * next, most likely idle, task, not the current one. Use 2710 * scx_bpf_kick_cpu() for deferred kicking. 2711 */ 2712 if (unlikely(!--nr_loops)) { 2713 scx_bpf_kick_cpu(cpu_of(rq), 0); 2714 break; 2715 } 2716 } while (dspc->nr_tasks); 2717 2718 no_tasks: 2719 /* 2720 * Didn't find another task to run. Keep running @prev unless 2721 * %SCX_OPS_ENQ_LAST is in effect. 2722 */ 2723 if ((prev->scx.flags & SCX_TASK_QUEUED) && 2724 (!static_branch_unlikely(&scx_ops_enq_last) || 2725 scx_rq_bypassing(rq))) { 2726 rq->scx.flags |= SCX_RQ_BAL_KEEP; 2727 goto has_tasks; 2728 } 2729 rq->scx.flags &= ~SCX_RQ_IN_BALANCE; 2730 return false; 2731 2732 has_tasks: 2733 rq->scx.flags &= ~SCX_RQ_IN_BALANCE; 2734 return true; 2735 } 2736 2737 static int balance_scx(struct rq *rq, struct task_struct *prev, 2738 struct rq_flags *rf) 2739 { 2740 int ret; 2741 2742 rq_unpin_lock(rq, rf); 2743 2744 ret = balance_one(rq, prev); 2745 2746 #ifdef CONFIG_SCHED_SMT 2747 /* 2748 * When core-sched is enabled, this ops.balance() call will be followed 2749 * by pick_task_scx() on this CPU and the SMT siblings. Balance the 2750 * siblings too. 2751 */ 2752 if (sched_core_enabled(rq)) { 2753 const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq)); 2754 int scpu; 2755 2756 for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) { 2757 struct rq *srq = cpu_rq(scpu); 2758 struct task_struct *sprev = srq->curr; 2759 2760 WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq)); 2761 update_rq_clock(srq); 2762 balance_one(srq, sprev); 2763 } 2764 } 2765 #endif 2766 rq_repin_lock(rq, rf); 2767 2768 return ret; 2769 } 2770 2771 static void process_ddsp_deferred_locals(struct rq *rq) 2772 { 2773 struct task_struct *p; 2774 2775 lockdep_assert_rq_held(rq); 2776 2777 /* 2778 * Now that @rq can be unlocked, execute the deferred enqueueing of 2779 * tasks directly dispatched to the local DSQs of other CPUs. See 2780 * direct_dispatch(). Keep popping from the head instead of using 2781 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq 2782 * temporarily. 2783 */ 2784 while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals, 2785 struct task_struct, scx.dsq_list.node))) { 2786 struct scx_dispatch_q *dsq; 2787 2788 list_del_init(&p->scx.dsq_list.node); 2789 2790 dsq = find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p); 2791 if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL)) 2792 dispatch_to_local_dsq(rq, dsq, p, p->scx.ddsp_enq_flags); 2793 } 2794 } 2795 2796 static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first) 2797 { 2798 if (p->scx.flags & SCX_TASK_QUEUED) { 2799 /* 2800 * Core-sched might decide to execute @p before it is 2801 * dispatched. Call ops_dequeue() to notify the BPF scheduler. 2802 */ 2803 ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC); 2804 dispatch_dequeue(rq, p); 2805 } 2806 2807 p->se.exec_start = rq_clock_task(rq); 2808 2809 /* see dequeue_task_scx() on why we skip when !QUEUED */ 2810 if (SCX_HAS_OP(running) && (p->scx.flags & SCX_TASK_QUEUED)) 2811 SCX_CALL_OP_TASK(SCX_KF_REST, running, p); 2812 2813 clr_task_runnable(p, true); 2814 2815 /* 2816 * @p is getting newly scheduled or got kicked after someone updated its 2817 * slice. Refresh whether tick can be stopped. See scx_can_stop_tick(). 2818 */ 2819 if ((p->scx.slice == SCX_SLICE_INF) != 2820 (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { 2821 if (p->scx.slice == SCX_SLICE_INF) 2822 rq->scx.flags |= SCX_RQ_CAN_STOP_TICK; 2823 else 2824 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; 2825 2826 sched_update_tick_dependency(rq); 2827 2828 /* 2829 * For now, let's refresh the load_avgs just when transitioning 2830 * in and out of nohz. In the future, we might want to add a 2831 * mechanism which calls the following periodically on 2832 * tick-stopped CPUs. 2833 */ 2834 update_other_load_avgs(rq); 2835 } 2836 } 2837 2838 static enum scx_cpu_preempt_reason 2839 preempt_reason_from_class(const struct sched_class *class) 2840 { 2841 #ifdef CONFIG_SMP 2842 if (class == &stop_sched_class) 2843 return SCX_CPU_PREEMPT_STOP; 2844 #endif 2845 if (class == &dl_sched_class) 2846 return SCX_CPU_PREEMPT_DL; 2847 if (class == &rt_sched_class) 2848 return SCX_CPU_PREEMPT_RT; 2849 return SCX_CPU_PREEMPT_UNKNOWN; 2850 } 2851 2852 static void switch_class(struct rq *rq, struct task_struct *next) 2853 { 2854 const struct sched_class *next_class = next->sched_class; 2855 2856 #ifdef CONFIG_SMP 2857 /* 2858 * Pairs with the smp_load_acquire() issued by a CPU in 2859 * kick_cpus_irq_workfn() who is waiting for this CPU to perform a 2860 * resched. 2861 */ 2862 smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1); 2863 #endif 2864 if (!static_branch_unlikely(&scx_ops_cpu_preempt)) 2865 return; 2866 2867 /* 2868 * The callback is conceptually meant to convey that the CPU is no 2869 * longer under the control of SCX. Therefore, don't invoke the callback 2870 * if the next class is below SCX (in which case the BPF scheduler has 2871 * actively decided not to schedule any tasks on the CPU). 2872 */ 2873 if (sched_class_above(&ext_sched_class, next_class)) 2874 return; 2875 2876 /* 2877 * At this point we know that SCX was preempted by a higher priority 2878 * sched_class, so invoke the ->cpu_release() callback if we have not 2879 * done so already. We only send the callback once between SCX being 2880 * preempted, and it regaining control of the CPU. 2881 * 2882 * ->cpu_release() complements ->cpu_acquire(), which is emitted the 2883 * next time that balance_scx() is invoked. 2884 */ 2885 if (!rq->scx.cpu_released) { 2886 if (SCX_HAS_OP(cpu_release)) { 2887 struct scx_cpu_release_args args = { 2888 .reason = preempt_reason_from_class(next_class), 2889 .task = next, 2890 }; 2891 2892 SCX_CALL_OP(SCX_KF_CPU_RELEASE, 2893 cpu_release, cpu_of(rq), &args); 2894 } 2895 rq->scx.cpu_released = true; 2896 } 2897 } 2898 2899 static void put_prev_task_scx(struct rq *rq, struct task_struct *p, 2900 struct task_struct *next) 2901 { 2902 update_curr_scx(rq); 2903 2904 /* see dequeue_task_scx() on why we skip when !QUEUED */ 2905 if (SCX_HAS_OP(stopping) && (p->scx.flags & SCX_TASK_QUEUED)) 2906 SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true); 2907 2908 if (p->scx.flags & SCX_TASK_QUEUED) { 2909 set_task_runnable(rq, p); 2910 2911 /* 2912 * If @p has slice left and is being put, @p is getting 2913 * preempted by a higher priority scheduler class or core-sched 2914 * forcing a different task. Leave it at the head of the local 2915 * DSQ. 2916 */ 2917 if (p->scx.slice && !scx_rq_bypassing(rq)) { 2918 dispatch_enqueue(&rq->scx.local_dsq, p, SCX_ENQ_HEAD); 2919 return; 2920 } 2921 2922 /* 2923 * If @p is runnable but we're about to enter a lower 2924 * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell 2925 * ops.enqueue() that @p is the only one available for this cpu, 2926 * which should trigger an explicit follow-up scheduling event. 2927 */ 2928 if (sched_class_above(&ext_sched_class, next->sched_class)) { 2929 WARN_ON_ONCE(!static_branch_unlikely(&scx_ops_enq_last)); 2930 do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); 2931 } else { 2932 do_enqueue_task(rq, p, 0, -1); 2933 } 2934 } 2935 2936 if (next && next->sched_class != &ext_sched_class) 2937 switch_class(rq, next); 2938 } 2939 2940 static struct task_struct *first_local_task(struct rq *rq) 2941 { 2942 return list_first_entry_or_null(&rq->scx.local_dsq.list, 2943 struct task_struct, scx.dsq_list.node); 2944 } 2945 2946 static struct task_struct *pick_task_scx(struct rq *rq) 2947 { 2948 struct task_struct *prev = rq->curr; 2949 struct task_struct *p; 2950 2951 /* 2952 * If balance_scx() is telling us to keep running @prev, replenish slice 2953 * if necessary and keep running @prev. Otherwise, pop the first one 2954 * from the local DSQ. 2955 * 2956 * WORKAROUND: 2957 * 2958 * %SCX_RQ_BAL_KEEP should be set iff $prev is on SCX as it must just 2959 * have gone through balance_scx(). Unfortunately, there currently is a 2960 * bug where fair could say yes on balance() but no on pick_task(), 2961 * which then ends up calling pick_task_scx() without preceding 2962 * balance_scx(). 2963 * 2964 * For now, ignore cases where $prev is not on SCX. This isn't great and 2965 * can theoretically lead to stalls. However, for switch_all cases, this 2966 * happens only while a BPF scheduler is being loaded or unloaded, and, 2967 * for partial cases, fair will likely keep triggering this CPU. 2968 * 2969 * Once fair is fixed, restore WARN_ON_ONCE(). 2970 */ 2971 if ((rq->scx.flags & SCX_RQ_BAL_KEEP) && 2972 prev->sched_class == &ext_sched_class) { 2973 p = prev; 2974 if (!p->scx.slice) 2975 p->scx.slice = SCX_SLICE_DFL; 2976 } else { 2977 p = first_local_task(rq); 2978 if (!p) 2979 return NULL; 2980 2981 if (unlikely(!p->scx.slice)) { 2982 if (!scx_rq_bypassing(rq) && !scx_warned_zero_slice) { 2983 printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n", 2984 p->comm, p->pid, __func__); 2985 scx_warned_zero_slice = true; 2986 } 2987 p->scx.slice = SCX_SLICE_DFL; 2988 } 2989 } 2990 2991 return p; 2992 } 2993 2994 #ifdef CONFIG_SCHED_CORE 2995 /** 2996 * scx_prio_less - Task ordering for core-sched 2997 * @a: task A 2998 * @b: task B 2999 * 3000 * Core-sched is implemented as an additional scheduling layer on top of the 3001 * usual sched_class'es and needs to find out the expected task ordering. For 3002 * SCX, core-sched calls this function to interrogate the task ordering. 3003 * 3004 * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used 3005 * to implement the default task ordering. The older the timestamp, the higher 3006 * prority the task - the global FIFO ordering matching the default scheduling 3007 * behavior. 3008 * 3009 * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to 3010 * implement FIFO ordering within each local DSQ. See pick_task_scx(). 3011 */ 3012 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, 3013 bool in_fi) 3014 { 3015 /* 3016 * The const qualifiers are dropped from task_struct pointers when 3017 * calling ops.core_sched_before(). Accesses are controlled by the 3018 * verifier. 3019 */ 3020 if (SCX_HAS_OP(core_sched_before) && !scx_rq_bypassing(task_rq(a))) 3021 return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before, 3022 (struct task_struct *)a, 3023 (struct task_struct *)b); 3024 else 3025 return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); 3026 } 3027 #endif /* CONFIG_SCHED_CORE */ 3028 3029 #ifdef CONFIG_SMP 3030 3031 static bool test_and_clear_cpu_idle(int cpu) 3032 { 3033 #ifdef CONFIG_SCHED_SMT 3034 /* 3035 * SMT mask should be cleared whether we can claim @cpu or not. The SMT 3036 * cluster is not wholly idle either way. This also prevents 3037 * scx_pick_idle_cpu() from getting caught in an infinite loop. 3038 */ 3039 if (sched_smt_active()) { 3040 const struct cpumask *smt = cpu_smt_mask(cpu); 3041 3042 /* 3043 * If offline, @cpu is not its own sibling and 3044 * scx_pick_idle_cpu() can get caught in an infinite loop as 3045 * @cpu is never cleared from idle_masks.smt. Ensure that @cpu 3046 * is eventually cleared. 3047 */ 3048 if (cpumask_intersects(smt, idle_masks.smt)) 3049 cpumask_andnot(idle_masks.smt, idle_masks.smt, smt); 3050 else if (cpumask_test_cpu(cpu, idle_masks.smt)) 3051 __cpumask_clear_cpu(cpu, idle_masks.smt); 3052 } 3053 #endif 3054 return cpumask_test_and_clear_cpu(cpu, idle_masks.cpu); 3055 } 3056 3057 static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags) 3058 { 3059 int cpu; 3060 3061 retry: 3062 if (sched_smt_active()) { 3063 cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed); 3064 if (cpu < nr_cpu_ids) 3065 goto found; 3066 3067 if (flags & SCX_PICK_IDLE_CORE) 3068 return -EBUSY; 3069 } 3070 3071 cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed); 3072 if (cpu >= nr_cpu_ids) 3073 return -EBUSY; 3074 3075 found: 3076 if (test_and_clear_cpu_idle(cpu)) 3077 return cpu; 3078 else 3079 goto retry; 3080 } 3081 3082 static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, 3083 u64 wake_flags, bool *found) 3084 { 3085 s32 cpu; 3086 3087 *found = false; 3088 3089 /* 3090 * If WAKE_SYNC, the waker's local DSQ is empty, and the system is 3091 * under utilized, wake up @p to the local DSQ of the waker. Checking 3092 * only for an empty local DSQ is insufficient as it could give the 3093 * wakee an unfair advantage when the system is oversaturated. 3094 * Checking only for the presence of idle CPUs is also insufficient as 3095 * the local DSQ of the waker could have tasks piled up on it even if 3096 * there is an idle core elsewhere on the system. 3097 */ 3098 cpu = smp_processor_id(); 3099 if ((wake_flags & SCX_WAKE_SYNC) && 3100 !cpumask_empty(idle_masks.cpu) && !(current->flags & PF_EXITING) && 3101 cpu_rq(cpu)->scx.local_dsq.nr == 0) { 3102 if (cpumask_test_cpu(cpu, p->cpus_ptr)) 3103 goto cpu_found; 3104 } 3105 3106 /* 3107 * If CPU has SMT, any wholly idle CPU is likely a better pick than 3108 * partially idle @prev_cpu. 3109 */ 3110 if (sched_smt_active()) { 3111 if (cpumask_test_cpu(prev_cpu, idle_masks.smt) && 3112 test_and_clear_cpu_idle(prev_cpu)) { 3113 cpu = prev_cpu; 3114 goto cpu_found; 3115 } 3116 3117 cpu = scx_pick_idle_cpu(p->cpus_ptr, SCX_PICK_IDLE_CORE); 3118 if (cpu >= 0) 3119 goto cpu_found; 3120 } 3121 3122 if (test_and_clear_cpu_idle(prev_cpu)) { 3123 cpu = prev_cpu; 3124 goto cpu_found; 3125 } 3126 3127 cpu = scx_pick_idle_cpu(p->cpus_ptr, 0); 3128 if (cpu >= 0) 3129 goto cpu_found; 3130 3131 return prev_cpu; 3132 3133 cpu_found: 3134 *found = true; 3135 return cpu; 3136 } 3137 3138 static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags) 3139 { 3140 /* 3141 * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it 3142 * can be a good migration opportunity with low cache and memory 3143 * footprint. Returning a CPU different than @prev_cpu triggers 3144 * immediate rq migration. However, for SCX, as the current rq 3145 * association doesn't dictate where the task is going to run, this 3146 * doesn't fit well. If necessary, we can later add a dedicated method 3147 * which can decide to preempt self to force it through the regular 3148 * scheduling path. 3149 */ 3150 if (unlikely(wake_flags & WF_EXEC)) 3151 return prev_cpu; 3152 3153 if (SCX_HAS_OP(select_cpu) && !scx_rq_bypassing(task_rq(p))) { 3154 s32 cpu; 3155 struct task_struct **ddsp_taskp; 3156 3157 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task); 3158 WARN_ON_ONCE(*ddsp_taskp); 3159 *ddsp_taskp = p; 3160 3161 cpu = SCX_CALL_OP_TASK_RET(SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU, 3162 select_cpu, p, prev_cpu, wake_flags); 3163 *ddsp_taskp = NULL; 3164 if (ops_cpu_valid(cpu, "from ops.select_cpu()")) 3165 return cpu; 3166 else 3167 return prev_cpu; 3168 } else { 3169 bool found; 3170 s32 cpu; 3171 3172 cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, &found); 3173 if (found) { 3174 p->scx.slice = SCX_SLICE_DFL; 3175 p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL; 3176 } 3177 return cpu; 3178 } 3179 } 3180 3181 static void task_woken_scx(struct rq *rq, struct task_struct *p) 3182 { 3183 run_deferred(rq); 3184 } 3185 3186 static void set_cpus_allowed_scx(struct task_struct *p, 3187 struct affinity_context *ac) 3188 { 3189 set_cpus_allowed_common(p, ac); 3190 3191 /* 3192 * The effective cpumask is stored in @p->cpus_ptr which may temporarily 3193 * differ from the configured one in @p->cpus_mask. Always tell the bpf 3194 * scheduler the effective one. 3195 * 3196 * Fine-grained memory write control is enforced by BPF making the const 3197 * designation pointless. Cast it away when calling the operation. 3198 */ 3199 if (SCX_HAS_OP(set_cpumask)) 3200 SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, 3201 (struct cpumask *)p->cpus_ptr); 3202 } 3203 3204 static void reset_idle_masks(void) 3205 { 3206 /* 3207 * Consider all online cpus idle. Should converge to the actual state 3208 * quickly. 3209 */ 3210 cpumask_copy(idle_masks.cpu, cpu_online_mask); 3211 cpumask_copy(idle_masks.smt, cpu_online_mask); 3212 } 3213 3214 void __scx_update_idle(struct rq *rq, bool idle) 3215 { 3216 int cpu = cpu_of(rq); 3217 3218 if (SCX_HAS_OP(update_idle) && !scx_rq_bypassing(rq)) { 3219 SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle); 3220 if (!static_branch_unlikely(&scx_builtin_idle_enabled)) 3221 return; 3222 } 3223 3224 if (idle) 3225 cpumask_set_cpu(cpu, idle_masks.cpu); 3226 else 3227 cpumask_clear_cpu(cpu, idle_masks.cpu); 3228 3229 #ifdef CONFIG_SCHED_SMT 3230 if (sched_smt_active()) { 3231 const struct cpumask *smt = cpu_smt_mask(cpu); 3232 3233 if (idle) { 3234 /* 3235 * idle_masks.smt handling is racy but that's fine as 3236 * it's only for optimization and self-correcting. 3237 */ 3238 for_each_cpu(cpu, smt) { 3239 if (!cpumask_test_cpu(cpu, idle_masks.cpu)) 3240 return; 3241 } 3242 cpumask_or(idle_masks.smt, idle_masks.smt, smt); 3243 } else { 3244 cpumask_andnot(idle_masks.smt, idle_masks.smt, smt); 3245 } 3246 } 3247 #endif 3248 } 3249 3250 static void handle_hotplug(struct rq *rq, bool online) 3251 { 3252 int cpu = cpu_of(rq); 3253 3254 atomic_long_inc(&scx_hotplug_seq); 3255 3256 if (online && SCX_HAS_OP(cpu_online)) 3257 SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_online, cpu); 3258 else if (!online && SCX_HAS_OP(cpu_offline)) 3259 SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_offline, cpu); 3260 else 3261 scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, 3262 "cpu %d going %s, exiting scheduler", cpu, 3263 online ? "online" : "offline"); 3264 } 3265 3266 void scx_rq_activate(struct rq *rq) 3267 { 3268 handle_hotplug(rq, true); 3269 } 3270 3271 void scx_rq_deactivate(struct rq *rq) 3272 { 3273 handle_hotplug(rq, false); 3274 } 3275 3276 static void rq_online_scx(struct rq *rq) 3277 { 3278 rq->scx.flags |= SCX_RQ_ONLINE; 3279 } 3280 3281 static void rq_offline_scx(struct rq *rq) 3282 { 3283 rq->scx.flags &= ~SCX_RQ_ONLINE; 3284 } 3285 3286 #else /* CONFIG_SMP */ 3287 3288 static bool test_and_clear_cpu_idle(int cpu) { return false; } 3289 static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags) { return -EBUSY; } 3290 static void reset_idle_masks(void) {} 3291 3292 #endif /* CONFIG_SMP */ 3293 3294 static bool check_rq_for_timeouts(struct rq *rq) 3295 { 3296 struct task_struct *p; 3297 struct rq_flags rf; 3298 bool timed_out = false; 3299 3300 rq_lock_irqsave(rq, &rf); 3301 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { 3302 unsigned long last_runnable = p->scx.runnable_at; 3303 3304 if (unlikely(time_after(jiffies, 3305 last_runnable + scx_watchdog_timeout))) { 3306 u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); 3307 3308 scx_ops_error_kind(SCX_EXIT_ERROR_STALL, 3309 "%s[%d] failed to run for %u.%03us", 3310 p->comm, p->pid, 3311 dur_ms / 1000, dur_ms % 1000); 3312 timed_out = true; 3313 break; 3314 } 3315 } 3316 rq_unlock_irqrestore(rq, &rf); 3317 3318 return timed_out; 3319 } 3320 3321 static void scx_watchdog_workfn(struct work_struct *work) 3322 { 3323 int cpu; 3324 3325 WRITE_ONCE(scx_watchdog_timestamp, jiffies); 3326 3327 for_each_online_cpu(cpu) { 3328 if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) 3329 break; 3330 3331 cond_resched(); 3332 } 3333 queue_delayed_work(system_unbound_wq, to_delayed_work(work), 3334 scx_watchdog_timeout / 2); 3335 } 3336 3337 void scx_tick(struct rq *rq) 3338 { 3339 unsigned long last_check; 3340 3341 if (!scx_enabled()) 3342 return; 3343 3344 last_check = READ_ONCE(scx_watchdog_timestamp); 3345 if (unlikely(time_after(jiffies, 3346 last_check + READ_ONCE(scx_watchdog_timeout)))) { 3347 u32 dur_ms = jiffies_to_msecs(jiffies - last_check); 3348 3349 scx_ops_error_kind(SCX_EXIT_ERROR_STALL, 3350 "watchdog failed to check in for %u.%03us", 3351 dur_ms / 1000, dur_ms % 1000); 3352 } 3353 3354 update_other_load_avgs(rq); 3355 } 3356 3357 static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) 3358 { 3359 update_curr_scx(rq); 3360 3361 /* 3362 * While disabling, always resched and refresh core-sched timestamp as 3363 * we can't trust the slice management or ops.core_sched_before(). 3364 */ 3365 if (scx_rq_bypassing(rq)) { 3366 curr->scx.slice = 0; 3367 touch_core_sched(rq, curr); 3368 } else if (SCX_HAS_OP(tick)) { 3369 SCX_CALL_OP(SCX_KF_REST, tick, curr); 3370 } 3371 3372 if (!curr->scx.slice) 3373 resched_curr(rq); 3374 } 3375 3376 #ifdef CONFIG_EXT_GROUP_SCHED 3377 static struct cgroup *tg_cgrp(struct task_group *tg) 3378 { 3379 /* 3380 * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup, 3381 * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the 3382 * root cgroup. 3383 */ 3384 if (tg && tg->css.cgroup) 3385 return tg->css.cgroup; 3386 else 3387 return &cgrp_dfl_root.cgrp; 3388 } 3389 3390 #define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg), 3391 3392 #else /* CONFIG_EXT_GROUP_SCHED */ 3393 3394 #define SCX_INIT_TASK_ARGS_CGROUP(tg) 3395 3396 #endif /* CONFIG_EXT_GROUP_SCHED */ 3397 3398 static enum scx_task_state scx_get_task_state(const struct task_struct *p) 3399 { 3400 return (p->scx.flags & SCX_TASK_STATE_MASK) >> SCX_TASK_STATE_SHIFT; 3401 } 3402 3403 static void scx_set_task_state(struct task_struct *p, enum scx_task_state state) 3404 { 3405 enum scx_task_state prev_state = scx_get_task_state(p); 3406 bool warn = false; 3407 3408 BUILD_BUG_ON(SCX_TASK_NR_STATES > (1 << SCX_TASK_STATE_BITS)); 3409 3410 switch (state) { 3411 case SCX_TASK_NONE: 3412 break; 3413 case SCX_TASK_INIT: 3414 warn = prev_state != SCX_TASK_NONE; 3415 break; 3416 case SCX_TASK_READY: 3417 warn = prev_state == SCX_TASK_NONE; 3418 break; 3419 case SCX_TASK_ENABLED: 3420 warn = prev_state != SCX_TASK_READY; 3421 break; 3422 default: 3423 warn = true; 3424 return; 3425 } 3426 3427 WARN_ONCE(warn, "sched_ext: Invalid task state transition %d -> %d for %s[%d]", 3428 prev_state, state, p->comm, p->pid); 3429 3430 p->scx.flags &= ~SCX_TASK_STATE_MASK; 3431 p->scx.flags |= state << SCX_TASK_STATE_SHIFT; 3432 } 3433 3434 static int scx_ops_init_task(struct task_struct *p, struct task_group *tg, bool fork) 3435 { 3436 int ret; 3437 3438 p->scx.disallow = false; 3439 3440 if (SCX_HAS_OP(init_task)) { 3441 struct scx_init_task_args args = { 3442 SCX_INIT_TASK_ARGS_CGROUP(tg) 3443 .fork = fork, 3444 }; 3445 3446 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init_task, p, &args); 3447 if (unlikely(ret)) { 3448 ret = ops_sanitize_err("init_task", ret); 3449 return ret; 3450 } 3451 } 3452 3453 scx_set_task_state(p, SCX_TASK_INIT); 3454 3455 if (p->scx.disallow) { 3456 if (!fork) { 3457 struct rq *rq; 3458 struct rq_flags rf; 3459 3460 rq = task_rq_lock(p, &rf); 3461 3462 /* 3463 * We're in the load path and @p->policy will be applied 3464 * right after. Reverting @p->policy here and rejecting 3465 * %SCHED_EXT transitions from scx_check_setscheduler() 3466 * guarantees that if ops.init_task() sets @p->disallow, 3467 * @p can never be in SCX. 3468 */ 3469 if (p->policy == SCHED_EXT) { 3470 p->policy = SCHED_NORMAL; 3471 atomic_long_inc(&scx_nr_rejected); 3472 } 3473 3474 task_rq_unlock(rq, p, &rf); 3475 } else if (p->policy == SCHED_EXT) { 3476 scx_ops_error("ops.init_task() set task->scx.disallow for %s[%d] during fork", 3477 p->comm, p->pid); 3478 } 3479 } 3480 3481 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; 3482 return 0; 3483 } 3484 3485 static void scx_ops_enable_task(struct task_struct *p) 3486 { 3487 u32 weight; 3488 3489 lockdep_assert_rq_held(task_rq(p)); 3490 3491 /* 3492 * Set the weight before calling ops.enable() so that the scheduler 3493 * doesn't see a stale value if they inspect the task struct. 3494 */ 3495 if (task_has_idle_policy(p)) 3496 weight = WEIGHT_IDLEPRIO; 3497 else 3498 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; 3499 3500 p->scx.weight = sched_weight_to_cgroup(weight); 3501 3502 if (SCX_HAS_OP(enable)) 3503 SCX_CALL_OP_TASK(SCX_KF_REST, enable, p); 3504 scx_set_task_state(p, SCX_TASK_ENABLED); 3505 3506 if (SCX_HAS_OP(set_weight)) 3507 SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx.weight); 3508 } 3509 3510 static void scx_ops_disable_task(struct task_struct *p) 3511 { 3512 lockdep_assert_rq_held(task_rq(p)); 3513 WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED); 3514 3515 if (SCX_HAS_OP(disable)) 3516 SCX_CALL_OP(SCX_KF_REST, disable, p); 3517 scx_set_task_state(p, SCX_TASK_READY); 3518 } 3519 3520 static void scx_ops_exit_task(struct task_struct *p) 3521 { 3522 struct scx_exit_task_args args = { 3523 .cancelled = false, 3524 }; 3525 3526 lockdep_assert_rq_held(task_rq(p)); 3527 3528 switch (scx_get_task_state(p)) { 3529 case SCX_TASK_NONE: 3530 return; 3531 case SCX_TASK_INIT: 3532 args.cancelled = true; 3533 break; 3534 case SCX_TASK_READY: 3535 break; 3536 case SCX_TASK_ENABLED: 3537 scx_ops_disable_task(p); 3538 break; 3539 default: 3540 WARN_ON_ONCE(true); 3541 return; 3542 } 3543 3544 if (SCX_HAS_OP(exit_task)) 3545 SCX_CALL_OP(SCX_KF_REST, exit_task, p, &args); 3546 scx_set_task_state(p, SCX_TASK_NONE); 3547 } 3548 3549 void init_scx_entity(struct sched_ext_entity *scx) 3550 { 3551 /* 3552 * init_idle() calls this function again after fork sequence is 3553 * complete. Don't touch ->tasks_node as it's already linked. 3554 */ 3555 memset(scx, 0, offsetof(struct sched_ext_entity, tasks_node)); 3556 3557 INIT_LIST_HEAD(&scx->dsq_list.node); 3558 RB_CLEAR_NODE(&scx->dsq_priq); 3559 scx->sticky_cpu = -1; 3560 scx->holding_cpu = -1; 3561 INIT_LIST_HEAD(&scx->runnable_node); 3562 scx->runnable_at = jiffies; 3563 scx->ddsp_dsq_id = SCX_DSQ_INVALID; 3564 scx->slice = SCX_SLICE_DFL; 3565 } 3566 3567 void scx_pre_fork(struct task_struct *p) 3568 { 3569 /* 3570 * BPF scheduler enable/disable paths want to be able to iterate and 3571 * update all tasks which can become complex when racing forks. As 3572 * enable/disable are very cold paths, let's use a percpu_rwsem to 3573 * exclude forks. 3574 */ 3575 percpu_down_read(&scx_fork_rwsem); 3576 } 3577 3578 int scx_fork(struct task_struct *p) 3579 { 3580 percpu_rwsem_assert_held(&scx_fork_rwsem); 3581 3582 if (scx_ops_init_task_enabled) 3583 return scx_ops_init_task(p, task_group(p), true); 3584 else 3585 return 0; 3586 } 3587 3588 void scx_post_fork(struct task_struct *p) 3589 { 3590 if (scx_ops_init_task_enabled) { 3591 scx_set_task_state(p, SCX_TASK_READY); 3592 3593 /* 3594 * Enable the task immediately if it's running on sched_ext. 3595 * Otherwise, it'll be enabled in switching_to_scx() if and 3596 * when it's ever configured to run with a SCHED_EXT policy. 3597 */ 3598 if (p->sched_class == &ext_sched_class) { 3599 struct rq_flags rf; 3600 struct rq *rq; 3601 3602 rq = task_rq_lock(p, &rf); 3603 scx_ops_enable_task(p); 3604 task_rq_unlock(rq, p, &rf); 3605 } 3606 } 3607 3608 spin_lock_irq(&scx_tasks_lock); 3609 list_add_tail(&p->scx.tasks_node, &scx_tasks); 3610 spin_unlock_irq(&scx_tasks_lock); 3611 3612 percpu_up_read(&scx_fork_rwsem); 3613 } 3614 3615 void scx_cancel_fork(struct task_struct *p) 3616 { 3617 if (scx_enabled()) { 3618 struct rq *rq; 3619 struct rq_flags rf; 3620 3621 rq = task_rq_lock(p, &rf); 3622 WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY); 3623 scx_ops_exit_task(p); 3624 task_rq_unlock(rq, p, &rf); 3625 } 3626 3627 percpu_up_read(&scx_fork_rwsem); 3628 } 3629 3630 void sched_ext_free(struct task_struct *p) 3631 { 3632 unsigned long flags; 3633 3634 spin_lock_irqsave(&scx_tasks_lock, flags); 3635 list_del_init(&p->scx.tasks_node); 3636 spin_unlock_irqrestore(&scx_tasks_lock, flags); 3637 3638 /* 3639 * @p is off scx_tasks and wholly ours. scx_ops_enable()'s READY -> 3640 * ENABLED transitions can't race us. Disable ops for @p. 3641 */ 3642 if (scx_get_task_state(p) != SCX_TASK_NONE) { 3643 struct rq_flags rf; 3644 struct rq *rq; 3645 3646 rq = task_rq_lock(p, &rf); 3647 scx_ops_exit_task(p); 3648 task_rq_unlock(rq, p, &rf); 3649 } 3650 } 3651 3652 static void reweight_task_scx(struct rq *rq, struct task_struct *p, 3653 const struct load_weight *lw) 3654 { 3655 lockdep_assert_rq_held(task_rq(p)); 3656 3657 p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight)); 3658 if (SCX_HAS_OP(set_weight)) 3659 SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx.weight); 3660 } 3661 3662 static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio) 3663 { 3664 } 3665 3666 static void switching_to_scx(struct rq *rq, struct task_struct *p) 3667 { 3668 scx_ops_enable_task(p); 3669 3670 /* 3671 * set_cpus_allowed_scx() is not called while @p is associated with a 3672 * different scheduler class. Keep the BPF scheduler up-to-date. 3673 */ 3674 if (SCX_HAS_OP(set_cpumask)) 3675 SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p, 3676 (struct cpumask *)p->cpus_ptr); 3677 } 3678 3679 static void switched_from_scx(struct rq *rq, struct task_struct *p) 3680 { 3681 scx_ops_disable_task(p); 3682 } 3683 3684 static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p,int wake_flags) {} 3685 static void switched_to_scx(struct rq *rq, struct task_struct *p) {} 3686 3687 int scx_check_setscheduler(struct task_struct *p, int policy) 3688 { 3689 lockdep_assert_rq_held(task_rq(p)); 3690 3691 /* if disallow, reject transitioning into SCX */ 3692 if (scx_enabled() && READ_ONCE(p->scx.disallow) && 3693 p->policy != policy && policy == SCHED_EXT) 3694 return -EACCES; 3695 3696 return 0; 3697 } 3698 3699 #ifdef CONFIG_NO_HZ_FULL 3700 bool scx_can_stop_tick(struct rq *rq) 3701 { 3702 struct task_struct *p = rq->curr; 3703 3704 if (scx_rq_bypassing(rq)) 3705 return false; 3706 3707 if (p->sched_class != &ext_sched_class) 3708 return true; 3709 3710 /* 3711 * @rq can dispatch from different DSQs, so we can't tell whether it 3712 * needs the tick or not by looking at nr_running. Allow stopping ticks 3713 * iff the BPF scheduler indicated so. See set_next_task_scx(). 3714 */ 3715 return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; 3716 } 3717 #endif 3718 3719 #ifdef CONFIG_EXT_GROUP_SCHED 3720 3721 DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_rwsem); 3722 static bool scx_cgroup_enabled; 3723 static bool cgroup_warned_missing_weight; 3724 static bool cgroup_warned_missing_idle; 3725 3726 static void scx_cgroup_warn_missing_weight(struct task_group *tg) 3727 { 3728 if (scx_ops_enable_state() == SCX_OPS_DISABLED || 3729 cgroup_warned_missing_weight) 3730 return; 3731 3732 if ((scx_ops.flags & SCX_OPS_HAS_CGROUP_WEIGHT) || !tg->css.parent) 3733 return; 3734 3735 pr_warn("sched_ext: \"%s\" does not implement cgroup cpu.weight\n", 3736 scx_ops.name); 3737 cgroup_warned_missing_weight = true; 3738 } 3739 3740 static void scx_cgroup_warn_missing_idle(struct task_group *tg) 3741 { 3742 if (!scx_cgroup_enabled || cgroup_warned_missing_idle) 3743 return; 3744 3745 if (!tg->idle) 3746 return; 3747 3748 pr_warn("sched_ext: \"%s\" does not implement cgroup cpu.idle\n", 3749 scx_ops.name); 3750 cgroup_warned_missing_idle = true; 3751 } 3752 3753 int scx_tg_online(struct task_group *tg) 3754 { 3755 int ret = 0; 3756 3757 WARN_ON_ONCE(tg->scx_flags & (SCX_TG_ONLINE | SCX_TG_INITED)); 3758 3759 percpu_down_read(&scx_cgroup_rwsem); 3760 3761 scx_cgroup_warn_missing_weight(tg); 3762 3763 if (scx_cgroup_enabled) { 3764 if (SCX_HAS_OP(cgroup_init)) { 3765 struct scx_cgroup_init_args args = 3766 { .weight = tg->scx_weight }; 3767 3768 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init, 3769 tg->css.cgroup, &args); 3770 if (ret) 3771 ret = ops_sanitize_err("cgroup_init", ret); 3772 } 3773 if (ret == 0) 3774 tg->scx_flags |= SCX_TG_ONLINE | SCX_TG_INITED; 3775 } else { 3776 tg->scx_flags |= SCX_TG_ONLINE; 3777 } 3778 3779 percpu_up_read(&scx_cgroup_rwsem); 3780 return ret; 3781 } 3782 3783 void scx_tg_offline(struct task_group *tg) 3784 { 3785 WARN_ON_ONCE(!(tg->scx_flags & SCX_TG_ONLINE)); 3786 3787 percpu_down_read(&scx_cgroup_rwsem); 3788 3789 if (SCX_HAS_OP(cgroup_exit) && (tg->scx_flags & SCX_TG_INITED)) 3790 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_exit, tg->css.cgroup); 3791 tg->scx_flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED); 3792 3793 percpu_up_read(&scx_cgroup_rwsem); 3794 } 3795 3796 int scx_cgroup_can_attach(struct cgroup_taskset *tset) 3797 { 3798 struct cgroup_subsys_state *css; 3799 struct task_struct *p; 3800 int ret; 3801 3802 /* released in scx_finish/cancel_attach() */ 3803 percpu_down_read(&scx_cgroup_rwsem); 3804 3805 if (!scx_cgroup_enabled) 3806 return 0; 3807 3808 cgroup_taskset_for_each(p, css, tset) { 3809 struct cgroup *from = tg_cgrp(task_group(p)); 3810 struct cgroup *to = tg_cgrp(css_tg(css)); 3811 3812 WARN_ON_ONCE(p->scx.cgrp_moving_from); 3813 3814 /* 3815 * sched_move_task() omits identity migrations. Let's match the 3816 * behavior so that ops.cgroup_prep_move() and ops.cgroup_move() 3817 * always match one-to-one. 3818 */ 3819 if (from == to) 3820 continue; 3821 3822 if (SCX_HAS_OP(cgroup_prep_move)) { 3823 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_prep_move, 3824 p, from, css->cgroup); 3825 if (ret) 3826 goto err; 3827 } 3828 3829 p->scx.cgrp_moving_from = from; 3830 } 3831 3832 return 0; 3833 3834 err: 3835 cgroup_taskset_for_each(p, css, tset) { 3836 if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from) 3837 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_cancel_move, p, 3838 p->scx.cgrp_moving_from, css->cgroup); 3839 p->scx.cgrp_moving_from = NULL; 3840 } 3841 3842 percpu_up_read(&scx_cgroup_rwsem); 3843 return ops_sanitize_err("cgroup_prep_move", ret); 3844 } 3845 3846 void scx_move_task(struct task_struct *p) 3847 { 3848 if (!scx_cgroup_enabled) 3849 return; 3850 3851 /* 3852 * We're called from sched_move_task() which handles both cgroup and 3853 * autogroup moves. Ignore the latter. 3854 * 3855 * Also ignore exiting tasks, because in the exit path tasks transition 3856 * from the autogroup to the root group, so task_group_is_autogroup() 3857 * alone isn't able to catch exiting autogroup tasks. This is safe for 3858 * cgroup_move(), because cgroup migrations never happen for PF_EXITING 3859 * tasks. 3860 */ 3861 if (task_group_is_autogroup(task_group(p)) || (p->flags & PF_EXITING)) 3862 return; 3863 3864 /* 3865 * @p must have ops.cgroup_prep_move() called on it and thus 3866 * cgrp_moving_from set. 3867 */ 3868 if (SCX_HAS_OP(cgroup_move) && !WARN_ON_ONCE(!p->scx.cgrp_moving_from)) 3869 SCX_CALL_OP_TASK(SCX_KF_UNLOCKED, cgroup_move, p, 3870 p->scx.cgrp_moving_from, tg_cgrp(task_group(p))); 3871 p->scx.cgrp_moving_from = NULL; 3872 } 3873 3874 void scx_cgroup_finish_attach(void) 3875 { 3876 percpu_up_read(&scx_cgroup_rwsem); 3877 } 3878 3879 void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) 3880 { 3881 struct cgroup_subsys_state *css; 3882 struct task_struct *p; 3883 3884 if (!scx_cgroup_enabled) 3885 goto out_unlock; 3886 3887 cgroup_taskset_for_each(p, css, tset) { 3888 if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from) 3889 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_cancel_move, p, 3890 p->scx.cgrp_moving_from, css->cgroup); 3891 p->scx.cgrp_moving_from = NULL; 3892 } 3893 out_unlock: 3894 percpu_up_read(&scx_cgroup_rwsem); 3895 } 3896 3897 void scx_group_set_weight(struct task_group *tg, unsigned long weight) 3898 { 3899 percpu_down_read(&scx_cgroup_rwsem); 3900 3901 if (scx_cgroup_enabled && tg->scx_weight != weight) { 3902 if (SCX_HAS_OP(cgroup_set_weight)) 3903 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_set_weight, 3904 tg_cgrp(tg), weight); 3905 tg->scx_weight = weight; 3906 } 3907 3908 percpu_up_read(&scx_cgroup_rwsem); 3909 } 3910 3911 void scx_group_set_idle(struct task_group *tg, bool idle) 3912 { 3913 percpu_down_read(&scx_cgroup_rwsem); 3914 scx_cgroup_warn_missing_idle(tg); 3915 percpu_up_read(&scx_cgroup_rwsem); 3916 } 3917 3918 static void scx_cgroup_lock(void) 3919 { 3920 percpu_down_write(&scx_cgroup_rwsem); 3921 } 3922 3923 static void scx_cgroup_unlock(void) 3924 { 3925 percpu_up_write(&scx_cgroup_rwsem); 3926 } 3927 3928 #else /* CONFIG_EXT_GROUP_SCHED */ 3929 3930 static inline void scx_cgroup_lock(void) {} 3931 static inline void scx_cgroup_unlock(void) {} 3932 3933 #endif /* CONFIG_EXT_GROUP_SCHED */ 3934 3935 /* 3936 * Omitted operations: 3937 * 3938 * - wakeup_preempt: NOOP as it isn't useful in the wakeup path because the task 3939 * isn't tied to the CPU at that point. Preemption is implemented by resetting 3940 * the victim task's slice to 0 and triggering reschedule on the target CPU. 3941 * 3942 * - migrate_task_rq: Unnecessary as task to cpu mapping is transient. 3943 * 3944 * - task_fork/dead: We need fork/dead notifications for all tasks regardless of 3945 * their current sched_class. Call them directly from sched core instead. 3946 */ 3947 DEFINE_SCHED_CLASS(ext) = { 3948 .enqueue_task = enqueue_task_scx, 3949 .dequeue_task = dequeue_task_scx, 3950 .yield_task = yield_task_scx, 3951 .yield_to_task = yield_to_task_scx, 3952 3953 .wakeup_preempt = wakeup_preempt_scx, 3954 3955 .balance = balance_scx, 3956 .pick_task = pick_task_scx, 3957 3958 .put_prev_task = put_prev_task_scx, 3959 .set_next_task = set_next_task_scx, 3960 3961 #ifdef CONFIG_SMP 3962 .select_task_rq = select_task_rq_scx, 3963 .task_woken = task_woken_scx, 3964 .set_cpus_allowed = set_cpus_allowed_scx, 3965 3966 .rq_online = rq_online_scx, 3967 .rq_offline = rq_offline_scx, 3968 #endif 3969 3970 .task_tick = task_tick_scx, 3971 3972 .switching_to = switching_to_scx, 3973 .switched_from = switched_from_scx, 3974 .switched_to = switched_to_scx, 3975 .reweight_task = reweight_task_scx, 3976 .prio_changed = prio_changed_scx, 3977 3978 .update_curr = update_curr_scx, 3979 3980 #ifdef CONFIG_UCLAMP_TASK 3981 .uclamp_enabled = 1, 3982 #endif 3983 }; 3984 3985 static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id) 3986 { 3987 memset(dsq, 0, sizeof(*dsq)); 3988 3989 raw_spin_lock_init(&dsq->lock); 3990 INIT_LIST_HEAD(&dsq->list); 3991 dsq->id = dsq_id; 3992 } 3993 3994 static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node) 3995 { 3996 struct scx_dispatch_q *dsq; 3997 int ret; 3998 3999 if (dsq_id & SCX_DSQ_FLAG_BUILTIN) 4000 return ERR_PTR(-EINVAL); 4001 4002 dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node); 4003 if (!dsq) 4004 return ERR_PTR(-ENOMEM); 4005 4006 init_dsq(dsq, dsq_id); 4007 4008 ret = rhashtable_insert_fast(&dsq_hash, &dsq->hash_node, 4009 dsq_hash_params); 4010 if (ret) { 4011 kfree(dsq); 4012 return ERR_PTR(ret); 4013 } 4014 return dsq; 4015 } 4016 4017 static void free_dsq_irq_workfn(struct irq_work *irq_work) 4018 { 4019 struct llist_node *to_free = llist_del_all(&dsqs_to_free); 4020 struct scx_dispatch_q *dsq, *tmp_dsq; 4021 4022 llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) 4023 kfree_rcu(dsq, rcu); 4024 } 4025 4026 static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn); 4027 4028 static void destroy_dsq(u64 dsq_id) 4029 { 4030 struct scx_dispatch_q *dsq; 4031 unsigned long flags; 4032 4033 rcu_read_lock(); 4034 4035 dsq = find_user_dsq(dsq_id); 4036 if (!dsq) 4037 goto out_unlock_rcu; 4038 4039 raw_spin_lock_irqsave(&dsq->lock, flags); 4040 4041 if (dsq->nr) { 4042 scx_ops_error("attempting to destroy in-use dsq 0x%016llx (nr=%u)", 4043 dsq->id, dsq->nr); 4044 goto out_unlock_dsq; 4045 } 4046 4047 if (rhashtable_remove_fast(&dsq_hash, &dsq->hash_node, dsq_hash_params)) 4048 goto out_unlock_dsq; 4049 4050 /* 4051 * Mark dead by invalidating ->id to prevent dispatch_enqueue() from 4052 * queueing more tasks. As this function can be called from anywhere, 4053 * freeing is bounced through an irq work to avoid nesting RCU 4054 * operations inside scheduler locks. 4055 */ 4056 dsq->id = SCX_DSQ_INVALID; 4057 llist_add(&dsq->free_node, &dsqs_to_free); 4058 irq_work_queue(&free_dsq_irq_work); 4059 4060 out_unlock_dsq: 4061 raw_spin_unlock_irqrestore(&dsq->lock, flags); 4062 out_unlock_rcu: 4063 rcu_read_unlock(); 4064 } 4065 4066 #ifdef CONFIG_EXT_GROUP_SCHED 4067 static void scx_cgroup_exit(void) 4068 { 4069 struct cgroup_subsys_state *css; 4070 4071 percpu_rwsem_assert_held(&scx_cgroup_rwsem); 4072 4073 scx_cgroup_enabled = false; 4074 4075 /* 4076 * scx_tg_on/offline() are excluded through scx_cgroup_rwsem. If we walk 4077 * cgroups and exit all the inited ones, all online cgroups are exited. 4078 */ 4079 rcu_read_lock(); 4080 css_for_each_descendant_post(css, &root_task_group.css) { 4081 struct task_group *tg = css_tg(css); 4082 4083 if (!(tg->scx_flags & SCX_TG_INITED)) 4084 continue; 4085 tg->scx_flags &= ~SCX_TG_INITED; 4086 4087 if (!scx_ops.cgroup_exit) 4088 continue; 4089 4090 if (WARN_ON_ONCE(!css_tryget(css))) 4091 continue; 4092 rcu_read_unlock(); 4093 4094 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_exit, css->cgroup); 4095 4096 rcu_read_lock(); 4097 css_put(css); 4098 } 4099 rcu_read_unlock(); 4100 } 4101 4102 static int scx_cgroup_init(void) 4103 { 4104 struct cgroup_subsys_state *css; 4105 int ret; 4106 4107 percpu_rwsem_assert_held(&scx_cgroup_rwsem); 4108 4109 cgroup_warned_missing_weight = false; 4110 cgroup_warned_missing_idle = false; 4111 4112 /* 4113 * scx_tg_on/offline() are excluded thorugh scx_cgroup_rwsem. If we walk 4114 * cgroups and init, all online cgroups are initialized. 4115 */ 4116 rcu_read_lock(); 4117 css_for_each_descendant_pre(css, &root_task_group.css) { 4118 struct task_group *tg = css_tg(css); 4119 struct scx_cgroup_init_args args = { .weight = tg->scx_weight }; 4120 4121 scx_cgroup_warn_missing_weight(tg); 4122 scx_cgroup_warn_missing_idle(tg); 4123 4124 if ((tg->scx_flags & 4125 (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE) 4126 continue; 4127 4128 if (!scx_ops.cgroup_init) { 4129 tg->scx_flags |= SCX_TG_INITED; 4130 continue; 4131 } 4132 4133 if (WARN_ON_ONCE(!css_tryget(css))) 4134 continue; 4135 rcu_read_unlock(); 4136 4137 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init, 4138 css->cgroup, &args); 4139 if (ret) { 4140 css_put(css); 4141 scx_ops_error("ops.cgroup_init() failed (%d)", ret); 4142 return ret; 4143 } 4144 tg->scx_flags |= SCX_TG_INITED; 4145 4146 rcu_read_lock(); 4147 css_put(css); 4148 } 4149 rcu_read_unlock(); 4150 4151 WARN_ON_ONCE(scx_cgroup_enabled); 4152 scx_cgroup_enabled = true; 4153 4154 return 0; 4155 } 4156 4157 #else 4158 static void scx_cgroup_exit(void) {} 4159 static int scx_cgroup_init(void) { return 0; } 4160 #endif 4161 4162 4163 /******************************************************************************** 4164 * Sysfs interface and ops enable/disable. 4165 */ 4166 4167 #define SCX_ATTR(_name) \ 4168 static struct kobj_attribute scx_attr_##_name = { \ 4169 .attr = { .name = __stringify(_name), .mode = 0444 }, \ 4170 .show = scx_attr_##_name##_show, \ 4171 } 4172 4173 static ssize_t scx_attr_state_show(struct kobject *kobj, 4174 struct kobj_attribute *ka, char *buf) 4175 { 4176 return sysfs_emit(buf, "%s\n", 4177 scx_ops_enable_state_str[scx_ops_enable_state()]); 4178 } 4179 SCX_ATTR(state); 4180 4181 static ssize_t scx_attr_switch_all_show(struct kobject *kobj, 4182 struct kobj_attribute *ka, char *buf) 4183 { 4184 return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all)); 4185 } 4186 SCX_ATTR(switch_all); 4187 4188 static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj, 4189 struct kobj_attribute *ka, char *buf) 4190 { 4191 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected)); 4192 } 4193 SCX_ATTR(nr_rejected); 4194 4195 static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj, 4196 struct kobj_attribute *ka, char *buf) 4197 { 4198 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq)); 4199 } 4200 SCX_ATTR(hotplug_seq); 4201 4202 static ssize_t scx_attr_enable_seq_show(struct kobject *kobj, 4203 struct kobj_attribute *ka, char *buf) 4204 { 4205 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq)); 4206 } 4207 SCX_ATTR(enable_seq); 4208 4209 static struct attribute *scx_global_attrs[] = { 4210 &scx_attr_state.attr, 4211 &scx_attr_switch_all.attr, 4212 &scx_attr_nr_rejected.attr, 4213 &scx_attr_hotplug_seq.attr, 4214 &scx_attr_enable_seq.attr, 4215 NULL, 4216 }; 4217 4218 static const struct attribute_group scx_global_attr_group = { 4219 .attrs = scx_global_attrs, 4220 }; 4221 4222 static void scx_kobj_release(struct kobject *kobj) 4223 { 4224 kfree(kobj); 4225 } 4226 4227 static ssize_t scx_attr_ops_show(struct kobject *kobj, 4228 struct kobj_attribute *ka, char *buf) 4229 { 4230 return sysfs_emit(buf, "%s\n", scx_ops.name); 4231 } 4232 SCX_ATTR(ops); 4233 4234 static struct attribute *scx_sched_attrs[] = { 4235 &scx_attr_ops.attr, 4236 NULL, 4237 }; 4238 ATTRIBUTE_GROUPS(scx_sched); 4239 4240 static const struct kobj_type scx_ktype = { 4241 .release = scx_kobj_release, 4242 .sysfs_ops = &kobj_sysfs_ops, 4243 .default_groups = scx_sched_groups, 4244 }; 4245 4246 static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) 4247 { 4248 return add_uevent_var(env, "SCXOPS=%s", scx_ops.name); 4249 } 4250 4251 static const struct kset_uevent_ops scx_uevent_ops = { 4252 .uevent = scx_uevent, 4253 }; 4254 4255 /* 4256 * Used by sched_fork() and __setscheduler_prio() to pick the matching 4257 * sched_class. dl/rt are already handled. 4258 */ 4259 bool task_should_scx(struct task_struct *p) 4260 { 4261 if (!scx_enabled() || 4262 unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING)) 4263 return false; 4264 if (READ_ONCE(scx_switching_all)) 4265 return true; 4266 return p->policy == SCHED_EXT; 4267 } 4268 4269 /** 4270 * scx_ops_bypass - [Un]bypass scx_ops and guarantee forward progress 4271 * 4272 * Bypassing guarantees that all runnable tasks make forward progress without 4273 * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might 4274 * be held by tasks that the BPF scheduler is forgetting to run, which 4275 * unfortunately also excludes toggling the static branches. 4276 * 4277 * Let's work around by overriding a couple ops and modifying behaviors based on 4278 * the DISABLING state and then cycling the queued tasks through dequeue/enqueue 4279 * to force global FIFO scheduling. 4280 * 4281 * - ops.select_cpu() is ignored and the default select_cpu() is used. 4282 * 4283 * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order. 4284 * %SCX_OPS_ENQ_LAST is also ignored. 4285 * 4286 * - ops.dispatch() is ignored. 4287 * 4288 * - balance_scx() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice 4289 * can't be trusted. Whenever a tick triggers, the running task is rotated to 4290 * the tail of the queue with core_sched_at touched. 4291 * 4292 * - pick_next_task() suppresses zero slice warning. 4293 * 4294 * - scx_bpf_kick_cpu() is disabled to avoid irq_work malfunction during PM 4295 * operations. 4296 * 4297 * - scx_prio_less() reverts to the default core_sched_at order. 4298 */ 4299 static void scx_ops_bypass(bool bypass) 4300 { 4301 int depth, cpu; 4302 4303 if (bypass) { 4304 depth = atomic_inc_return(&scx_ops_bypass_depth); 4305 WARN_ON_ONCE(depth <= 0); 4306 if (depth != 1) 4307 return; 4308 } else { 4309 depth = atomic_dec_return(&scx_ops_bypass_depth); 4310 WARN_ON_ONCE(depth < 0); 4311 if (depth != 0) 4312 return; 4313 } 4314 4315 /* 4316 * No task property is changing. We just need to make sure all currently 4317 * queued tasks are re-queued according to the new scx_rq_bypassing() 4318 * state. As an optimization, walk each rq's runnable_list instead of 4319 * the scx_tasks list. 4320 * 4321 * This function can't trust the scheduler and thus can't use 4322 * cpus_read_lock(). Walk all possible CPUs instead of online. 4323 */ 4324 for_each_possible_cpu(cpu) { 4325 struct rq *rq = cpu_rq(cpu); 4326 struct rq_flags rf; 4327 struct task_struct *p, *n; 4328 4329 rq_lock_irqsave(rq, &rf); 4330 4331 if (bypass) { 4332 WARN_ON_ONCE(rq->scx.flags & SCX_RQ_BYPASSING); 4333 rq->scx.flags |= SCX_RQ_BYPASSING; 4334 } else { 4335 WARN_ON_ONCE(!(rq->scx.flags & SCX_RQ_BYPASSING)); 4336 rq->scx.flags &= ~SCX_RQ_BYPASSING; 4337 } 4338 4339 /* 4340 * We need to guarantee that no tasks are on the BPF scheduler 4341 * while bypassing. Either we see enabled or the enable path 4342 * sees scx_rq_bypassing() before moving tasks to SCX. 4343 */ 4344 if (!scx_enabled()) { 4345 rq_unlock_irqrestore(rq, &rf); 4346 continue; 4347 } 4348 4349 /* 4350 * The use of list_for_each_entry_safe_reverse() is required 4351 * because each task is going to be removed from and added back 4352 * to the runnable_list during iteration. Because they're added 4353 * to the tail of the list, safe reverse iteration can still 4354 * visit all nodes. 4355 */ 4356 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, 4357 scx.runnable_node) { 4358 struct sched_enq_and_set_ctx ctx; 4359 4360 /* cycling deq/enq is enough, see the function comment */ 4361 sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx); 4362 sched_enq_and_set_task(&ctx); 4363 } 4364 4365 rq_unlock_irqrestore(rq, &rf); 4366 4367 /* resched to restore ticks and idle state */ 4368 resched_cpu(cpu); 4369 } 4370 } 4371 4372 static void free_exit_info(struct scx_exit_info *ei) 4373 { 4374 kfree(ei->dump); 4375 kfree(ei->msg); 4376 kfree(ei->bt); 4377 kfree(ei); 4378 } 4379 4380 static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len) 4381 { 4382 struct scx_exit_info *ei; 4383 4384 ei = kzalloc(sizeof(*ei), GFP_KERNEL); 4385 if (!ei) 4386 return NULL; 4387 4388 ei->bt = kcalloc(SCX_EXIT_BT_LEN, sizeof(ei->bt[0]), GFP_KERNEL); 4389 ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL); 4390 ei->dump = kzalloc(exit_dump_len, GFP_KERNEL); 4391 4392 if (!ei->bt || !ei->msg || !ei->dump) { 4393 free_exit_info(ei); 4394 return NULL; 4395 } 4396 4397 return ei; 4398 } 4399 4400 static const char *scx_exit_reason(enum scx_exit_kind kind) 4401 { 4402 switch (kind) { 4403 case SCX_EXIT_UNREG: 4404 return "unregistered from user space"; 4405 case SCX_EXIT_UNREG_BPF: 4406 return "unregistered from BPF"; 4407 case SCX_EXIT_UNREG_KERN: 4408 return "unregistered from the main kernel"; 4409 case SCX_EXIT_SYSRQ: 4410 return "disabled by sysrq-S"; 4411 case SCX_EXIT_ERROR: 4412 return "runtime error"; 4413 case SCX_EXIT_ERROR_BPF: 4414 return "scx_bpf_error"; 4415 case SCX_EXIT_ERROR_STALL: 4416 return "runnable task stall"; 4417 default: 4418 return "<UNKNOWN>"; 4419 } 4420 } 4421 4422 static void scx_ops_disable_workfn(struct kthread_work *work) 4423 { 4424 struct scx_exit_info *ei = scx_exit_info; 4425 struct scx_task_iter sti; 4426 struct task_struct *p; 4427 struct rhashtable_iter rht_iter; 4428 struct scx_dispatch_q *dsq; 4429 int i, kind; 4430 4431 kind = atomic_read(&scx_exit_kind); 4432 while (true) { 4433 /* 4434 * NONE indicates that a new scx_ops has been registered since 4435 * disable was scheduled - don't kill the new ops. DONE 4436 * indicates that the ops has already been disabled. 4437 */ 4438 if (kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE) 4439 return; 4440 if (atomic_try_cmpxchg(&scx_exit_kind, &kind, SCX_EXIT_DONE)) 4441 break; 4442 } 4443 ei->kind = kind; 4444 ei->reason = scx_exit_reason(ei->kind); 4445 4446 /* guarantee forward progress by bypassing scx_ops */ 4447 scx_ops_bypass(true); 4448 4449 switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) { 4450 case SCX_OPS_DISABLING: 4451 WARN_ONCE(true, "sched_ext: duplicate disabling instance?"); 4452 break; 4453 case SCX_OPS_DISABLED: 4454 pr_warn("sched_ext: ops error detected without ops (%s)\n", 4455 scx_exit_info->msg); 4456 WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != 4457 SCX_OPS_DISABLING); 4458 goto done; 4459 default: 4460 break; 4461 } 4462 4463 /* 4464 * Here, every runnable task is guaranteed to make forward progress and 4465 * we can safely use blocking synchronization constructs. Actually 4466 * disable ops. 4467 */ 4468 mutex_lock(&scx_ops_enable_mutex); 4469 4470 static_branch_disable(&__scx_switched_all); 4471 WRITE_ONCE(scx_switching_all, false); 4472 4473 /* 4474 * Shut down cgroup support before tasks so that the cgroup attach path 4475 * doesn't race against scx_ops_exit_task(). 4476 */ 4477 scx_cgroup_lock(); 4478 scx_cgroup_exit(); 4479 scx_cgroup_unlock(); 4480 4481 /* 4482 * The BPF scheduler is going away. All tasks including %TASK_DEAD ones 4483 * must be switched out and exited synchronously. 4484 */ 4485 percpu_down_write(&scx_fork_rwsem); 4486 4487 scx_ops_init_task_enabled = false; 4488 4489 scx_task_iter_start(&sti); 4490 while ((p = scx_task_iter_next_locked(&sti))) { 4491 const struct sched_class *old_class = p->sched_class; 4492 struct sched_enq_and_set_ctx ctx; 4493 4494 sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx); 4495 4496 p->sched_class = __setscheduler_class(p, p->prio); 4497 check_class_changing(task_rq(p), p, old_class); 4498 4499 sched_enq_and_set_task(&ctx); 4500 4501 check_class_changed(task_rq(p), p, old_class, p->prio); 4502 scx_ops_exit_task(p); 4503 } 4504 scx_task_iter_stop(&sti); 4505 percpu_up_write(&scx_fork_rwsem); 4506 4507 /* no task is on scx, turn off all the switches and flush in-progress calls */ 4508 static_branch_disable(&__scx_ops_enabled); 4509 for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) 4510 static_branch_disable(&scx_has_op[i]); 4511 static_branch_disable(&scx_ops_enq_last); 4512 static_branch_disable(&scx_ops_enq_exiting); 4513 static_branch_disable(&scx_ops_cpu_preempt); 4514 static_branch_disable(&scx_builtin_idle_enabled); 4515 synchronize_rcu(); 4516 4517 if (ei->kind >= SCX_EXIT_ERROR) { 4518 pr_err("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", 4519 scx_ops.name, ei->reason); 4520 4521 if (ei->msg[0] != '\0') 4522 pr_err("sched_ext: %s: %s\n", scx_ops.name, ei->msg); 4523 #ifdef CONFIG_STACKTRACE 4524 stack_trace_print(ei->bt, ei->bt_len, 2); 4525 #endif 4526 } else { 4527 pr_info("sched_ext: BPF scheduler \"%s\" disabled (%s)\n", 4528 scx_ops.name, ei->reason); 4529 } 4530 4531 if (scx_ops.exit) 4532 SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei); 4533 4534 cancel_delayed_work_sync(&scx_watchdog_work); 4535 4536 /* 4537 * Delete the kobject from the hierarchy eagerly in addition to just 4538 * dropping a reference. Otherwise, if the object is deleted 4539 * asynchronously, sysfs could observe an object of the same name still 4540 * in the hierarchy when another scheduler is loaded. 4541 */ 4542 kobject_del(scx_root_kobj); 4543 kobject_put(scx_root_kobj); 4544 scx_root_kobj = NULL; 4545 4546 memset(&scx_ops, 0, sizeof(scx_ops)); 4547 4548 rhashtable_walk_enter(&dsq_hash, &rht_iter); 4549 do { 4550 rhashtable_walk_start(&rht_iter); 4551 4552 while ((dsq = rhashtable_walk_next(&rht_iter)) && !IS_ERR(dsq)) 4553 destroy_dsq(dsq->id); 4554 4555 rhashtable_walk_stop(&rht_iter); 4556 } while (dsq == ERR_PTR(-EAGAIN)); 4557 rhashtable_walk_exit(&rht_iter); 4558 4559 free_percpu(scx_dsp_ctx); 4560 scx_dsp_ctx = NULL; 4561 scx_dsp_max_batch = 0; 4562 4563 free_exit_info(scx_exit_info); 4564 scx_exit_info = NULL; 4565 4566 mutex_unlock(&scx_ops_enable_mutex); 4567 4568 WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != 4569 SCX_OPS_DISABLING); 4570 done: 4571 scx_ops_bypass(false); 4572 } 4573 4574 static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn); 4575 4576 static void schedule_scx_ops_disable_work(void) 4577 { 4578 struct kthread_worker *helper = READ_ONCE(scx_ops_helper); 4579 4580 /* 4581 * We may be called spuriously before the first bpf_sched_ext_reg(). If 4582 * scx_ops_helper isn't set up yet, there's nothing to do. 4583 */ 4584 if (helper) 4585 kthread_queue_work(helper, &scx_ops_disable_work); 4586 } 4587 4588 static void scx_ops_disable(enum scx_exit_kind kind) 4589 { 4590 int none = SCX_EXIT_NONE; 4591 4592 if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) 4593 kind = SCX_EXIT_ERROR; 4594 4595 atomic_try_cmpxchg(&scx_exit_kind, &none, kind); 4596 4597 schedule_scx_ops_disable_work(); 4598 } 4599 4600 static void dump_newline(struct seq_buf *s) 4601 { 4602 trace_sched_ext_dump(""); 4603 4604 /* @s may be zero sized and seq_buf triggers WARN if so */ 4605 if (s->size) 4606 seq_buf_putc(s, '\n'); 4607 } 4608 4609 static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...) 4610 { 4611 va_list args; 4612 4613 #ifdef CONFIG_TRACEPOINTS 4614 if (trace_sched_ext_dump_enabled()) { 4615 /* protected by scx_dump_state()::dump_lock */ 4616 static char line_buf[SCX_EXIT_MSG_LEN]; 4617 4618 va_start(args, fmt); 4619 vscnprintf(line_buf, sizeof(line_buf), fmt, args); 4620 va_end(args); 4621 4622 trace_sched_ext_dump(line_buf); 4623 } 4624 #endif 4625 /* @s may be zero sized and seq_buf triggers WARN if so */ 4626 if (s->size) { 4627 va_start(args, fmt); 4628 seq_buf_vprintf(s, fmt, args); 4629 va_end(args); 4630 4631 seq_buf_putc(s, '\n'); 4632 } 4633 } 4634 4635 static void dump_stack_trace(struct seq_buf *s, const char *prefix, 4636 const unsigned long *bt, unsigned int len) 4637 { 4638 unsigned int i; 4639 4640 for (i = 0; i < len; i++) 4641 dump_line(s, "%s%pS", prefix, (void *)bt[i]); 4642 } 4643 4644 static void ops_dump_init(struct seq_buf *s, const char *prefix) 4645 { 4646 struct scx_dump_data *dd = &scx_dump_data; 4647 4648 lockdep_assert_irqs_disabled(); 4649 4650 dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */ 4651 dd->first = true; 4652 dd->cursor = 0; 4653 dd->s = s; 4654 dd->prefix = prefix; 4655 } 4656 4657 static void ops_dump_flush(void) 4658 { 4659 struct scx_dump_data *dd = &scx_dump_data; 4660 char *line = dd->buf.line; 4661 4662 if (!dd->cursor) 4663 return; 4664 4665 /* 4666 * There's something to flush and this is the first line. Insert a blank 4667 * line to distinguish ops dump. 4668 */ 4669 if (dd->first) { 4670 dump_newline(dd->s); 4671 dd->first = false; 4672 } 4673 4674 /* 4675 * There may be multiple lines in $line. Scan and emit each line 4676 * separately. 4677 */ 4678 while (true) { 4679 char *end = line; 4680 char c; 4681 4682 while (*end != '\n' && *end != '\0') 4683 end++; 4684 4685 /* 4686 * If $line overflowed, it may not have newline at the end. 4687 * Always emit with a newline. 4688 */ 4689 c = *end; 4690 *end = '\0'; 4691 dump_line(dd->s, "%s%s", dd->prefix, line); 4692 if (c == '\0') 4693 break; 4694 4695 /* move to the next line */ 4696 end++; 4697 if (*end == '\0') 4698 break; 4699 line = end; 4700 } 4701 4702 dd->cursor = 0; 4703 } 4704 4705 static void ops_dump_exit(void) 4706 { 4707 ops_dump_flush(); 4708 scx_dump_data.cpu = -1; 4709 } 4710 4711 static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx, 4712 struct task_struct *p, char marker) 4713 { 4714 static unsigned long bt[SCX_EXIT_BT_LEN]; 4715 char dsq_id_buf[19] = "(n/a)"; 4716 unsigned long ops_state = atomic_long_read(&p->scx.ops_state); 4717 unsigned int bt_len = 0; 4718 4719 if (p->scx.dsq) 4720 scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx", 4721 (unsigned long long)p->scx.dsq->id); 4722 4723 dump_newline(s); 4724 dump_line(s, " %c%c %s[%d] %+ldms", 4725 marker, task_state_to_char(p), p->comm, p->pid, 4726 jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies)); 4727 dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu", 4728 scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK, 4729 p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK, 4730 ops_state >> SCX_OPSS_QSEQ_SHIFT); 4731 dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s dsq_vtime=%llu", 4732 p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf, 4733 p->scx.dsq_vtime); 4734 dump_line(s, " cpus=%*pb", cpumask_pr_args(p->cpus_ptr)); 4735 4736 if (SCX_HAS_OP(dump_task)) { 4737 ops_dump_init(s, " "); 4738 SCX_CALL_OP(SCX_KF_REST, dump_task, dctx, p); 4739 ops_dump_exit(); 4740 } 4741 4742 #ifdef CONFIG_STACKTRACE 4743 bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1); 4744 #endif 4745 if (bt_len) { 4746 dump_newline(s); 4747 dump_stack_trace(s, " ", bt, bt_len); 4748 } 4749 } 4750 4751 static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len) 4752 { 4753 static DEFINE_SPINLOCK(dump_lock); 4754 static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n"; 4755 struct scx_dump_ctx dctx = { 4756 .kind = ei->kind, 4757 .exit_code = ei->exit_code, 4758 .reason = ei->reason, 4759 .at_ns = ktime_get_ns(), 4760 .at_jiffies = jiffies, 4761 }; 4762 struct seq_buf s; 4763 unsigned long flags; 4764 char *buf; 4765 int cpu; 4766 4767 spin_lock_irqsave(&dump_lock, flags); 4768 4769 seq_buf_init(&s, ei->dump, dump_len); 4770 4771 if (ei->kind == SCX_EXIT_NONE) { 4772 dump_line(&s, "Debug dump triggered by %s", ei->reason); 4773 } else { 4774 dump_line(&s, "%s[%d] triggered exit kind %d:", 4775 current->comm, current->pid, ei->kind); 4776 dump_line(&s, " %s (%s)", ei->reason, ei->msg); 4777 dump_newline(&s); 4778 dump_line(&s, "Backtrace:"); 4779 dump_stack_trace(&s, " ", ei->bt, ei->bt_len); 4780 } 4781 4782 if (SCX_HAS_OP(dump)) { 4783 ops_dump_init(&s, ""); 4784 SCX_CALL_OP(SCX_KF_UNLOCKED, dump, &dctx); 4785 ops_dump_exit(); 4786 } 4787 4788 dump_newline(&s); 4789 dump_line(&s, "CPU states"); 4790 dump_line(&s, "----------"); 4791 4792 for_each_possible_cpu(cpu) { 4793 struct rq *rq = cpu_rq(cpu); 4794 struct rq_flags rf; 4795 struct task_struct *p; 4796 struct seq_buf ns; 4797 size_t avail, used; 4798 bool idle; 4799 4800 rq_lock(rq, &rf); 4801 4802 idle = list_empty(&rq->scx.runnable_list) && 4803 rq->curr->sched_class == &idle_sched_class; 4804 4805 if (idle && !SCX_HAS_OP(dump_cpu)) 4806 goto next; 4807 4808 /* 4809 * We don't yet know whether ops.dump_cpu() will produce output 4810 * and we may want to skip the default CPU dump if it doesn't. 4811 * Use a nested seq_buf to generate the standard dump so that we 4812 * can decide whether to commit later. 4813 */ 4814 avail = seq_buf_get_buf(&s, &buf); 4815 seq_buf_init(&ns, buf, avail); 4816 4817 dump_newline(&ns); 4818 dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu pnt_seq=%lu", 4819 cpu, rq->scx.nr_running, rq->scx.flags, 4820 rq->scx.cpu_released, rq->scx.ops_qseq, 4821 rq->scx.pnt_seq); 4822 dump_line(&ns, " curr=%s[%d] class=%ps", 4823 rq->curr->comm, rq->curr->pid, 4824 rq->curr->sched_class); 4825 if (!cpumask_empty(rq->scx.cpus_to_kick)) 4826 dump_line(&ns, " cpus_to_kick : %*pb", 4827 cpumask_pr_args(rq->scx.cpus_to_kick)); 4828 if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle)) 4829 dump_line(&ns, " idle_to_kick : %*pb", 4830 cpumask_pr_args(rq->scx.cpus_to_kick_if_idle)); 4831 if (!cpumask_empty(rq->scx.cpus_to_preempt)) 4832 dump_line(&ns, " cpus_to_preempt: %*pb", 4833 cpumask_pr_args(rq->scx.cpus_to_preempt)); 4834 if (!cpumask_empty(rq->scx.cpus_to_wait)) 4835 dump_line(&ns, " cpus_to_wait : %*pb", 4836 cpumask_pr_args(rq->scx.cpus_to_wait)); 4837 4838 used = seq_buf_used(&ns); 4839 if (SCX_HAS_OP(dump_cpu)) { 4840 ops_dump_init(&ns, " "); 4841 SCX_CALL_OP(SCX_KF_REST, dump_cpu, &dctx, cpu, idle); 4842 ops_dump_exit(); 4843 } 4844 4845 /* 4846 * If idle && nothing generated by ops.dump_cpu(), there's 4847 * nothing interesting. Skip. 4848 */ 4849 if (idle && used == seq_buf_used(&ns)) 4850 goto next; 4851 4852 /* 4853 * $s may already have overflowed when $ns was created. If so, 4854 * calling commit on it will trigger BUG. 4855 */ 4856 if (avail) { 4857 seq_buf_commit(&s, seq_buf_used(&ns)); 4858 if (seq_buf_has_overflowed(&ns)) 4859 seq_buf_set_overflow(&s); 4860 } 4861 4862 if (rq->curr->sched_class == &ext_sched_class) 4863 scx_dump_task(&s, &dctx, rq->curr, '*'); 4864 4865 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) 4866 scx_dump_task(&s, &dctx, p, ' '); 4867 next: 4868 rq_unlock(rq, &rf); 4869 } 4870 4871 if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker)) 4872 memcpy(ei->dump + dump_len - sizeof(trunc_marker), 4873 trunc_marker, sizeof(trunc_marker)); 4874 4875 spin_unlock_irqrestore(&dump_lock, flags); 4876 } 4877 4878 static void scx_ops_error_irq_workfn(struct irq_work *irq_work) 4879 { 4880 struct scx_exit_info *ei = scx_exit_info; 4881 4882 if (ei->kind >= SCX_EXIT_ERROR) 4883 scx_dump_state(ei, scx_ops.exit_dump_len); 4884 4885 schedule_scx_ops_disable_work(); 4886 } 4887 4888 static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn); 4889 4890 static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind, 4891 s64 exit_code, 4892 const char *fmt, ...) 4893 { 4894 struct scx_exit_info *ei = scx_exit_info; 4895 int none = SCX_EXIT_NONE; 4896 va_list args; 4897 4898 if (!atomic_try_cmpxchg(&scx_exit_kind, &none, kind)) 4899 return; 4900 4901 ei->exit_code = exit_code; 4902 #ifdef CONFIG_STACKTRACE 4903 if (kind >= SCX_EXIT_ERROR) 4904 ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1); 4905 #endif 4906 va_start(args, fmt); 4907 vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args); 4908 va_end(args); 4909 4910 /* 4911 * Set ei->kind and ->reason for scx_dump_state(). They'll be set again 4912 * in scx_ops_disable_workfn(). 4913 */ 4914 ei->kind = kind; 4915 ei->reason = scx_exit_reason(ei->kind); 4916 4917 irq_work_queue(&scx_ops_error_irq_work); 4918 } 4919 4920 static struct kthread_worker *scx_create_rt_helper(const char *name) 4921 { 4922 struct kthread_worker *helper; 4923 4924 helper = kthread_create_worker(0, name); 4925 if (helper) 4926 sched_set_fifo(helper->task); 4927 return helper; 4928 } 4929 4930 static void check_hotplug_seq(const struct sched_ext_ops *ops) 4931 { 4932 unsigned long long global_hotplug_seq; 4933 4934 /* 4935 * If a hotplug event has occurred between when a scheduler was 4936 * initialized, and when we were able to attach, exit and notify user 4937 * space about it. 4938 */ 4939 if (ops->hotplug_seq) { 4940 global_hotplug_seq = atomic_long_read(&scx_hotplug_seq); 4941 if (ops->hotplug_seq != global_hotplug_seq) { 4942 scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, 4943 "expected hotplug seq %llu did not match actual %llu", 4944 ops->hotplug_seq, global_hotplug_seq); 4945 } 4946 } 4947 } 4948 4949 static int validate_ops(const struct sched_ext_ops *ops) 4950 { 4951 /* 4952 * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the 4953 * ops.enqueue() callback isn't implemented. 4954 */ 4955 if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) { 4956 scx_ops_error("SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented"); 4957 return -EINVAL; 4958 } 4959 4960 return 0; 4961 } 4962 4963 static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) 4964 { 4965 struct scx_task_iter sti; 4966 struct task_struct *p; 4967 unsigned long timeout; 4968 int i, cpu, node, ret; 4969 4970 if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), 4971 cpu_possible_mask)) { 4972 pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation"); 4973 return -EINVAL; 4974 } 4975 4976 mutex_lock(&scx_ops_enable_mutex); 4977 4978 if (!scx_ops_helper) { 4979 WRITE_ONCE(scx_ops_helper, 4980 scx_create_rt_helper("sched_ext_ops_helper")); 4981 if (!scx_ops_helper) { 4982 ret = -ENOMEM; 4983 goto err_unlock; 4984 } 4985 } 4986 4987 if (!global_dsqs) { 4988 struct scx_dispatch_q **dsqs; 4989 4990 dsqs = kcalloc(nr_node_ids, sizeof(dsqs[0]), GFP_KERNEL); 4991 if (!dsqs) { 4992 ret = -ENOMEM; 4993 goto err_unlock; 4994 } 4995 4996 for_each_node_state(node, N_POSSIBLE) { 4997 struct scx_dispatch_q *dsq; 4998 4999 dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node); 5000 if (!dsq) { 5001 for_each_node_state(node, N_POSSIBLE) 5002 kfree(dsqs[node]); 5003 kfree(dsqs); 5004 ret = -ENOMEM; 5005 goto err_unlock; 5006 } 5007 5008 init_dsq(dsq, SCX_DSQ_GLOBAL); 5009 dsqs[node] = dsq; 5010 } 5011 5012 global_dsqs = dsqs; 5013 } 5014 5015 if (scx_ops_enable_state() != SCX_OPS_DISABLED) { 5016 ret = -EBUSY; 5017 goto err_unlock; 5018 } 5019 5020 scx_root_kobj = kzalloc(sizeof(*scx_root_kobj), GFP_KERNEL); 5021 if (!scx_root_kobj) { 5022 ret = -ENOMEM; 5023 goto err_unlock; 5024 } 5025 5026 scx_root_kobj->kset = scx_kset; 5027 ret = kobject_init_and_add(scx_root_kobj, &scx_ktype, NULL, "root"); 5028 if (ret < 0) 5029 goto err; 5030 5031 scx_exit_info = alloc_exit_info(ops->exit_dump_len); 5032 if (!scx_exit_info) { 5033 ret = -ENOMEM; 5034 goto err_del; 5035 } 5036 5037 /* 5038 * Set scx_ops, transition to ENABLING and clear exit info to arm the 5039 * disable path. Failure triggers full disabling from here on. 5040 */ 5041 scx_ops = *ops; 5042 5043 WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_ENABLING) != 5044 SCX_OPS_DISABLED); 5045 5046 atomic_set(&scx_exit_kind, SCX_EXIT_NONE); 5047 scx_warned_zero_slice = false; 5048 5049 atomic_long_set(&scx_nr_rejected, 0); 5050 5051 for_each_possible_cpu(cpu) 5052 cpu_rq(cpu)->scx.cpuperf_target = SCX_CPUPERF_ONE; 5053 5054 /* 5055 * Keep CPUs stable during enable so that the BPF scheduler can track 5056 * online CPUs by watching ->on/offline_cpu() after ->init(). 5057 */ 5058 cpus_read_lock(); 5059 5060 if (scx_ops.init) { 5061 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init); 5062 if (ret) { 5063 ret = ops_sanitize_err("init", ret); 5064 cpus_read_unlock(); 5065 scx_ops_error("ops.init() failed (%d)", ret); 5066 goto err_disable; 5067 } 5068 } 5069 5070 for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) 5071 if (((void (**)(void))ops)[i]) 5072 static_branch_enable_cpuslocked(&scx_has_op[i]); 5073 5074 check_hotplug_seq(ops); 5075 cpus_read_unlock(); 5076 5077 ret = validate_ops(ops); 5078 if (ret) 5079 goto err_disable; 5080 5081 WARN_ON_ONCE(scx_dsp_ctx); 5082 scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH; 5083 scx_dsp_ctx = __alloc_percpu(struct_size_t(struct scx_dsp_ctx, buf, 5084 scx_dsp_max_batch), 5085 __alignof__(struct scx_dsp_ctx)); 5086 if (!scx_dsp_ctx) { 5087 ret = -ENOMEM; 5088 goto err_disable; 5089 } 5090 5091 if (ops->timeout_ms) 5092 timeout = msecs_to_jiffies(ops->timeout_ms); 5093 else 5094 timeout = SCX_WATCHDOG_MAX_TIMEOUT; 5095 5096 WRITE_ONCE(scx_watchdog_timeout, timeout); 5097 WRITE_ONCE(scx_watchdog_timestamp, jiffies); 5098 queue_delayed_work(system_unbound_wq, &scx_watchdog_work, 5099 scx_watchdog_timeout / 2); 5100 5101 /* 5102 * Once __scx_ops_enabled is set, %current can be switched to SCX 5103 * anytime. This can lead to stalls as some BPF schedulers (e.g. 5104 * userspace scheduling) may not function correctly before all tasks are 5105 * switched. Init in bypass mode to guarantee forward progress. 5106 */ 5107 scx_ops_bypass(true); 5108 5109 for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++) 5110 if (((void (**)(void))ops)[i]) 5111 static_branch_enable(&scx_has_op[i]); 5112 5113 if (ops->flags & SCX_OPS_ENQ_LAST) 5114 static_branch_enable(&scx_ops_enq_last); 5115 5116 if (ops->flags & SCX_OPS_ENQ_EXITING) 5117 static_branch_enable(&scx_ops_enq_exiting); 5118 if (scx_ops.cpu_acquire || scx_ops.cpu_release) 5119 static_branch_enable(&scx_ops_cpu_preempt); 5120 5121 if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) { 5122 reset_idle_masks(); 5123 static_branch_enable(&scx_builtin_idle_enabled); 5124 } else { 5125 static_branch_disable(&scx_builtin_idle_enabled); 5126 } 5127 5128 /* 5129 * Lock out forks, cgroup on/offlining and moves before opening the 5130 * floodgate so that they don't wander into the operations prematurely. 5131 */ 5132 percpu_down_write(&scx_fork_rwsem); 5133 5134 WARN_ON_ONCE(scx_ops_init_task_enabled); 5135 scx_ops_init_task_enabled = true; 5136 5137 /* 5138 * Enable ops for every task. Fork is excluded by scx_fork_rwsem 5139 * preventing new tasks from being added. No need to exclude tasks 5140 * leaving as sched_ext_free() can handle both prepped and enabled 5141 * tasks. Prep all tasks first and then enable them with preemption 5142 * disabled. 5143 * 5144 * All cgroups should be initialized before scx_ops_init_task() so that 5145 * the BPF scheduler can reliably track each task's cgroup membership 5146 * from scx_ops_init_task(). Lock out cgroup on/offlining and task 5147 * migrations while tasks are being initialized so that 5148 * scx_cgroup_can_attach() never sees uninitialized tasks. 5149 */ 5150 scx_cgroup_lock(); 5151 ret = scx_cgroup_init(); 5152 if (ret) 5153 goto err_disable_unlock_all; 5154 5155 scx_task_iter_start(&sti); 5156 while ((p = scx_task_iter_next_locked(&sti))) { 5157 /* 5158 * @p may already be dead, have lost all its usages counts and 5159 * be waiting for RCU grace period before being freed. @p can't 5160 * be initialized for SCX in such cases and should be ignored. 5161 */ 5162 if (!tryget_task_struct(p)) 5163 continue; 5164 5165 scx_task_iter_unlock(&sti); 5166 5167 ret = scx_ops_init_task(p, task_group(p), false); 5168 if (ret) { 5169 put_task_struct(p); 5170 scx_task_iter_relock(&sti); 5171 scx_task_iter_stop(&sti); 5172 scx_ops_error("ops.init_task() failed (%d) for %s[%d]", 5173 ret, p->comm, p->pid); 5174 goto err_disable_unlock_all; 5175 } 5176 5177 scx_set_task_state(p, SCX_TASK_READY); 5178 5179 put_task_struct(p); 5180 scx_task_iter_relock(&sti); 5181 } 5182 scx_task_iter_stop(&sti); 5183 scx_cgroup_unlock(); 5184 percpu_up_write(&scx_fork_rwsem); 5185 5186 /* 5187 * All tasks are READY. It's safe to turn on scx_enabled() and switch 5188 * all eligible tasks. 5189 */ 5190 WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); 5191 static_branch_enable(&__scx_ops_enabled); 5192 5193 /* 5194 * We're fully committed and can't fail. The task READY -> ENABLED 5195 * transitions here are synchronized against sched_ext_free() through 5196 * scx_tasks_lock. 5197 */ 5198 percpu_down_write(&scx_fork_rwsem); 5199 scx_task_iter_start(&sti); 5200 while ((p = scx_task_iter_next_locked(&sti))) { 5201 const struct sched_class *old_class = p->sched_class; 5202 struct sched_enq_and_set_ctx ctx; 5203 5204 sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx); 5205 5206 p->scx.slice = SCX_SLICE_DFL; 5207 p->sched_class = __setscheduler_class(p, p->prio); 5208 check_class_changing(task_rq(p), p, old_class); 5209 5210 sched_enq_and_set_task(&ctx); 5211 5212 check_class_changed(task_rq(p), p, old_class, p->prio); 5213 } 5214 scx_task_iter_stop(&sti); 5215 percpu_up_write(&scx_fork_rwsem); 5216 5217 scx_ops_bypass(false); 5218 5219 if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) { 5220 WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE); 5221 goto err_disable; 5222 } 5223 5224 if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) 5225 static_branch_enable(&__scx_switched_all); 5226 5227 pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", 5228 scx_ops.name, scx_switched_all() ? "" : " (partial)"); 5229 kobject_uevent(scx_root_kobj, KOBJ_ADD); 5230 mutex_unlock(&scx_ops_enable_mutex); 5231 5232 atomic_long_inc(&scx_enable_seq); 5233 5234 return 0; 5235 5236 err_del: 5237 kobject_del(scx_root_kobj); 5238 err: 5239 kobject_put(scx_root_kobj); 5240 scx_root_kobj = NULL; 5241 if (scx_exit_info) { 5242 free_exit_info(scx_exit_info); 5243 scx_exit_info = NULL; 5244 } 5245 err_unlock: 5246 mutex_unlock(&scx_ops_enable_mutex); 5247 return ret; 5248 5249 err_disable_unlock_all: 5250 scx_cgroup_unlock(); 5251 percpu_up_write(&scx_fork_rwsem); 5252 scx_ops_bypass(false); 5253 err_disable: 5254 mutex_unlock(&scx_ops_enable_mutex); 5255 /* 5256 * Returning an error code here would not pass all the error information 5257 * to userspace. Record errno using scx_ops_error() for cases 5258 * scx_ops_error() wasn't already invoked and exit indicating success so 5259 * that the error is notified through ops.exit() with all the details. 5260 * 5261 * Flush scx_ops_disable_work to ensure that error is reported before 5262 * init completion. 5263 */ 5264 scx_ops_error("scx_ops_enable() failed (%d)", ret); 5265 kthread_flush_work(&scx_ops_disable_work); 5266 return 0; 5267 } 5268 5269 5270 /******************************************************************************** 5271 * bpf_struct_ops plumbing. 5272 */ 5273 #include <linux/bpf_verifier.h> 5274 #include <linux/bpf.h> 5275 #include <linux/btf.h> 5276 5277 extern struct btf *btf_vmlinux; 5278 static const struct btf_type *task_struct_type; 5279 static u32 task_struct_type_id; 5280 5281 static bool set_arg_maybe_null(const char *op, int arg_n, int off, int size, 5282 enum bpf_access_type type, 5283 const struct bpf_prog *prog, 5284 struct bpf_insn_access_aux *info) 5285 { 5286 struct btf *btf = bpf_get_btf_vmlinux(); 5287 const struct bpf_struct_ops_desc *st_ops_desc; 5288 const struct btf_member *member; 5289 const struct btf_type *t; 5290 u32 btf_id, member_idx; 5291 const char *mname; 5292 5293 /* struct_ops op args are all sequential, 64-bit numbers */ 5294 if (off != arg_n * sizeof(__u64)) 5295 return false; 5296 5297 /* btf_id should be the type id of struct sched_ext_ops */ 5298 btf_id = prog->aux->attach_btf_id; 5299 st_ops_desc = bpf_struct_ops_find(btf, btf_id); 5300 if (!st_ops_desc) 5301 return false; 5302 5303 /* BTF type of struct sched_ext_ops */ 5304 t = st_ops_desc->type; 5305 5306 member_idx = prog->expected_attach_type; 5307 if (member_idx >= btf_type_vlen(t)) 5308 return false; 5309 5310 /* 5311 * Get the member name of this struct_ops program, which corresponds to 5312 * a field in struct sched_ext_ops. For example, the member name of the 5313 * dispatch struct_ops program (callback) is "dispatch". 5314 */ 5315 member = &btf_type_member(t)[member_idx]; 5316 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 5317 5318 if (!strcmp(mname, op)) { 5319 /* 5320 * The value is a pointer to a type (struct task_struct) given 5321 * by a BTF ID (PTR_TO_BTF_ID). It is trusted (PTR_TRUSTED), 5322 * however, can be a NULL (PTR_MAYBE_NULL). The BPF program 5323 * should check the pointer to make sure it is not NULL before 5324 * using it, or the verifier will reject the program. 5325 * 5326 * Longer term, this is something that should be addressed by 5327 * BTF, and be fully contained within the verifier. 5328 */ 5329 info->reg_type = PTR_MAYBE_NULL | PTR_TO_BTF_ID | PTR_TRUSTED; 5330 info->btf = btf_vmlinux; 5331 info->btf_id = task_struct_type_id; 5332 5333 return true; 5334 } 5335 5336 return false; 5337 } 5338 5339 static bool bpf_scx_is_valid_access(int off, int size, 5340 enum bpf_access_type type, 5341 const struct bpf_prog *prog, 5342 struct bpf_insn_access_aux *info) 5343 { 5344 if (type != BPF_READ) 5345 return false; 5346 if (set_arg_maybe_null("dispatch", 1, off, size, type, prog, info) || 5347 set_arg_maybe_null("yield", 1, off, size, type, prog, info)) 5348 return true; 5349 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 5350 return false; 5351 if (off % size != 0) 5352 return false; 5353 5354 return btf_ctx_access(off, size, type, prog, info); 5355 } 5356 5357 static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log, 5358 const struct bpf_reg_state *reg, int off, 5359 int size) 5360 { 5361 const struct btf_type *t; 5362 5363 t = btf_type_by_id(reg->btf, reg->btf_id); 5364 if (t == task_struct_type) { 5365 if (off >= offsetof(struct task_struct, scx.slice) && 5366 off + size <= offsetofend(struct task_struct, scx.slice)) 5367 return SCALAR_VALUE; 5368 if (off >= offsetof(struct task_struct, scx.dsq_vtime) && 5369 off + size <= offsetofend(struct task_struct, scx.dsq_vtime)) 5370 return SCALAR_VALUE; 5371 if (off >= offsetof(struct task_struct, scx.disallow) && 5372 off + size <= offsetofend(struct task_struct, scx.disallow)) 5373 return SCALAR_VALUE; 5374 } 5375 5376 return -EACCES; 5377 } 5378 5379 static const struct bpf_func_proto * 5380 bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 5381 { 5382 switch (func_id) { 5383 case BPF_FUNC_task_storage_get: 5384 return &bpf_task_storage_get_proto; 5385 case BPF_FUNC_task_storage_delete: 5386 return &bpf_task_storage_delete_proto; 5387 default: 5388 return bpf_base_func_proto(func_id, prog); 5389 } 5390 } 5391 5392 static const struct bpf_verifier_ops bpf_scx_verifier_ops = { 5393 .get_func_proto = bpf_scx_get_func_proto, 5394 .is_valid_access = bpf_scx_is_valid_access, 5395 .btf_struct_access = bpf_scx_btf_struct_access, 5396 }; 5397 5398 static int bpf_scx_init_member(const struct btf_type *t, 5399 const struct btf_member *member, 5400 void *kdata, const void *udata) 5401 { 5402 const struct sched_ext_ops *uops = udata; 5403 struct sched_ext_ops *ops = kdata; 5404 u32 moff = __btf_member_bit_offset(t, member) / 8; 5405 int ret; 5406 5407 switch (moff) { 5408 case offsetof(struct sched_ext_ops, dispatch_max_batch): 5409 if (*(u32 *)(udata + moff) > INT_MAX) 5410 return -E2BIG; 5411 ops->dispatch_max_batch = *(u32 *)(udata + moff); 5412 return 1; 5413 case offsetof(struct sched_ext_ops, flags): 5414 if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS) 5415 return -EINVAL; 5416 ops->flags = *(u64 *)(udata + moff); 5417 return 1; 5418 case offsetof(struct sched_ext_ops, name): 5419 ret = bpf_obj_name_cpy(ops->name, uops->name, 5420 sizeof(ops->name)); 5421 if (ret < 0) 5422 return ret; 5423 if (ret == 0) 5424 return -EINVAL; 5425 return 1; 5426 case offsetof(struct sched_ext_ops, timeout_ms): 5427 if (msecs_to_jiffies(*(u32 *)(udata + moff)) > 5428 SCX_WATCHDOG_MAX_TIMEOUT) 5429 return -E2BIG; 5430 ops->timeout_ms = *(u32 *)(udata + moff); 5431 return 1; 5432 case offsetof(struct sched_ext_ops, exit_dump_len): 5433 ops->exit_dump_len = 5434 *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN; 5435 return 1; 5436 case offsetof(struct sched_ext_ops, hotplug_seq): 5437 ops->hotplug_seq = *(u64 *)(udata + moff); 5438 return 1; 5439 } 5440 5441 return 0; 5442 } 5443 5444 static int bpf_scx_check_member(const struct btf_type *t, 5445 const struct btf_member *member, 5446 const struct bpf_prog *prog) 5447 { 5448 u32 moff = __btf_member_bit_offset(t, member) / 8; 5449 5450 switch (moff) { 5451 case offsetof(struct sched_ext_ops, init_task): 5452 #ifdef CONFIG_EXT_GROUP_SCHED 5453 case offsetof(struct sched_ext_ops, cgroup_init): 5454 case offsetof(struct sched_ext_ops, cgroup_exit): 5455 case offsetof(struct sched_ext_ops, cgroup_prep_move): 5456 #endif 5457 case offsetof(struct sched_ext_ops, cpu_online): 5458 case offsetof(struct sched_ext_ops, cpu_offline): 5459 case offsetof(struct sched_ext_ops, init): 5460 case offsetof(struct sched_ext_ops, exit): 5461 break; 5462 default: 5463 if (prog->sleepable) 5464 return -EINVAL; 5465 } 5466 5467 return 0; 5468 } 5469 5470 static int bpf_scx_reg(void *kdata, struct bpf_link *link) 5471 { 5472 return scx_ops_enable(kdata, link); 5473 } 5474 5475 static void bpf_scx_unreg(void *kdata, struct bpf_link *link) 5476 { 5477 scx_ops_disable(SCX_EXIT_UNREG); 5478 kthread_flush_work(&scx_ops_disable_work); 5479 } 5480 5481 static int bpf_scx_init(struct btf *btf) 5482 { 5483 s32 type_id; 5484 5485 type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT); 5486 if (type_id < 0) 5487 return -EINVAL; 5488 task_struct_type = btf_type_by_id(btf, type_id); 5489 task_struct_type_id = type_id; 5490 5491 return 0; 5492 } 5493 5494 static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link) 5495 { 5496 /* 5497 * sched_ext does not support updating the actively-loaded BPF 5498 * scheduler, as registering a BPF scheduler can always fail if the 5499 * scheduler returns an error code for e.g. ops.init(), ops.init_task(), 5500 * etc. Similarly, we can always race with unregistration happening 5501 * elsewhere, such as with sysrq. 5502 */ 5503 return -EOPNOTSUPP; 5504 } 5505 5506 static int bpf_scx_validate(void *kdata) 5507 { 5508 return 0; 5509 } 5510 5511 static s32 select_cpu_stub(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; } 5512 static void enqueue_stub(struct task_struct *p, u64 enq_flags) {} 5513 static void dequeue_stub(struct task_struct *p, u64 enq_flags) {} 5514 static void dispatch_stub(s32 prev_cpu, struct task_struct *p) {} 5515 static void tick_stub(struct task_struct *p) {} 5516 static void runnable_stub(struct task_struct *p, u64 enq_flags) {} 5517 static void running_stub(struct task_struct *p) {} 5518 static void stopping_stub(struct task_struct *p, bool runnable) {} 5519 static void quiescent_stub(struct task_struct *p, u64 deq_flags) {} 5520 static bool yield_stub(struct task_struct *from, struct task_struct *to) { return false; } 5521 static bool core_sched_before_stub(struct task_struct *a, struct task_struct *b) { return false; } 5522 static void set_weight_stub(struct task_struct *p, u32 weight) {} 5523 static void set_cpumask_stub(struct task_struct *p, const struct cpumask *mask) {} 5524 static void update_idle_stub(s32 cpu, bool idle) {} 5525 static void cpu_acquire_stub(s32 cpu, struct scx_cpu_acquire_args *args) {} 5526 static void cpu_release_stub(s32 cpu, struct scx_cpu_release_args *args) {} 5527 static s32 init_task_stub(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; } 5528 static void exit_task_stub(struct task_struct *p, struct scx_exit_task_args *args) {} 5529 static void enable_stub(struct task_struct *p) {} 5530 static void disable_stub(struct task_struct *p) {} 5531 #ifdef CONFIG_EXT_GROUP_SCHED 5532 static s32 cgroup_init_stub(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; } 5533 static void cgroup_exit_stub(struct cgroup *cgrp) {} 5534 static s32 cgroup_prep_move_stub(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; } 5535 static void cgroup_move_stub(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} 5536 static void cgroup_cancel_move_stub(struct task_struct *p, struct cgroup *from, struct cgroup *to) {} 5537 static void cgroup_set_weight_stub(struct cgroup *cgrp, u32 weight) {} 5538 #endif 5539 static void cpu_online_stub(s32 cpu) {} 5540 static void cpu_offline_stub(s32 cpu) {} 5541 static s32 init_stub(void) { return -EINVAL; } 5542 static void exit_stub(struct scx_exit_info *info) {} 5543 static void dump_stub(struct scx_dump_ctx *ctx) {} 5544 static void dump_cpu_stub(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {} 5545 static void dump_task_stub(struct scx_dump_ctx *ctx, struct task_struct *p) {} 5546 5547 static struct sched_ext_ops __bpf_ops_sched_ext_ops = { 5548 .select_cpu = select_cpu_stub, 5549 .enqueue = enqueue_stub, 5550 .dequeue = dequeue_stub, 5551 .dispatch = dispatch_stub, 5552 .tick = tick_stub, 5553 .runnable = runnable_stub, 5554 .running = running_stub, 5555 .stopping = stopping_stub, 5556 .quiescent = quiescent_stub, 5557 .yield = yield_stub, 5558 .core_sched_before = core_sched_before_stub, 5559 .set_weight = set_weight_stub, 5560 .set_cpumask = set_cpumask_stub, 5561 .update_idle = update_idle_stub, 5562 .cpu_acquire = cpu_acquire_stub, 5563 .cpu_release = cpu_release_stub, 5564 .init_task = init_task_stub, 5565 .exit_task = exit_task_stub, 5566 .enable = enable_stub, 5567 .disable = disable_stub, 5568 #ifdef CONFIG_EXT_GROUP_SCHED 5569 .cgroup_init = cgroup_init_stub, 5570 .cgroup_exit = cgroup_exit_stub, 5571 .cgroup_prep_move = cgroup_prep_move_stub, 5572 .cgroup_move = cgroup_move_stub, 5573 .cgroup_cancel_move = cgroup_cancel_move_stub, 5574 .cgroup_set_weight = cgroup_set_weight_stub, 5575 #endif 5576 .cpu_online = cpu_online_stub, 5577 .cpu_offline = cpu_offline_stub, 5578 .init = init_stub, 5579 .exit = exit_stub, 5580 .dump = dump_stub, 5581 .dump_cpu = dump_cpu_stub, 5582 .dump_task = dump_task_stub, 5583 }; 5584 5585 static struct bpf_struct_ops bpf_sched_ext_ops = { 5586 .verifier_ops = &bpf_scx_verifier_ops, 5587 .reg = bpf_scx_reg, 5588 .unreg = bpf_scx_unreg, 5589 .check_member = bpf_scx_check_member, 5590 .init_member = bpf_scx_init_member, 5591 .init = bpf_scx_init, 5592 .update = bpf_scx_update, 5593 .validate = bpf_scx_validate, 5594 .name = "sched_ext_ops", 5595 .owner = THIS_MODULE, 5596 .cfi_stubs = &__bpf_ops_sched_ext_ops 5597 }; 5598 5599 5600 /******************************************************************************** 5601 * System integration and init. 5602 */ 5603 5604 static void sysrq_handle_sched_ext_reset(u8 key) 5605 { 5606 if (scx_ops_helper) 5607 scx_ops_disable(SCX_EXIT_SYSRQ); 5608 else 5609 pr_info("sched_ext: BPF scheduler not yet used\n"); 5610 } 5611 5612 static const struct sysrq_key_op sysrq_sched_ext_reset_op = { 5613 .handler = sysrq_handle_sched_ext_reset, 5614 .help_msg = "reset-sched-ext(S)", 5615 .action_msg = "Disable sched_ext and revert all tasks to CFS", 5616 .enable_mask = SYSRQ_ENABLE_RTNICE, 5617 }; 5618 5619 static void sysrq_handle_sched_ext_dump(u8 key) 5620 { 5621 struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" }; 5622 5623 if (scx_enabled()) 5624 scx_dump_state(&ei, 0); 5625 } 5626 5627 static const struct sysrq_key_op sysrq_sched_ext_dump_op = { 5628 .handler = sysrq_handle_sched_ext_dump, 5629 .help_msg = "dump-sched-ext(D)", 5630 .action_msg = "Trigger sched_ext debug dump", 5631 .enable_mask = SYSRQ_ENABLE_RTNICE, 5632 }; 5633 5634 static bool can_skip_idle_kick(struct rq *rq) 5635 { 5636 lockdep_assert_rq_held(rq); 5637 5638 /* 5639 * We can skip idle kicking if @rq is going to go through at least one 5640 * full SCX scheduling cycle before going idle. Just checking whether 5641 * curr is not idle is insufficient because we could be racing 5642 * balance_one() trying to pull the next task from a remote rq, which 5643 * may fail, and @rq may become idle afterwards. 5644 * 5645 * The race window is small and we don't and can't guarantee that @rq is 5646 * only kicked while idle anyway. Skip only when sure. 5647 */ 5648 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE); 5649 } 5650 5651 static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *pseqs) 5652 { 5653 struct rq *rq = cpu_rq(cpu); 5654 struct scx_rq *this_scx = &this_rq->scx; 5655 bool should_wait = false; 5656 unsigned long flags; 5657 5658 raw_spin_rq_lock_irqsave(rq, flags); 5659 5660 /* 5661 * During CPU hotplug, a CPU may depend on kicking itself to make 5662 * forward progress. Allow kicking self regardless of online state. 5663 */ 5664 if (cpu_online(cpu) || cpu == cpu_of(this_rq)) { 5665 if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) { 5666 if (rq->curr->sched_class == &ext_sched_class) 5667 rq->curr->scx.slice = 0; 5668 cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); 5669 } 5670 5671 if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) { 5672 pseqs[cpu] = rq->scx.pnt_seq; 5673 should_wait = true; 5674 } 5675 5676 resched_curr(rq); 5677 } else { 5678 cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt); 5679 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); 5680 } 5681 5682 raw_spin_rq_unlock_irqrestore(rq, flags); 5683 5684 return should_wait; 5685 } 5686 5687 static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq) 5688 { 5689 struct rq *rq = cpu_rq(cpu); 5690 unsigned long flags; 5691 5692 raw_spin_rq_lock_irqsave(rq, flags); 5693 5694 if (!can_skip_idle_kick(rq) && 5695 (cpu_online(cpu) || cpu == cpu_of(this_rq))) 5696 resched_curr(rq); 5697 5698 raw_spin_rq_unlock_irqrestore(rq, flags); 5699 } 5700 5701 static void kick_cpus_irq_workfn(struct irq_work *irq_work) 5702 { 5703 struct rq *this_rq = this_rq(); 5704 struct scx_rq *this_scx = &this_rq->scx; 5705 unsigned long *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs); 5706 bool should_wait = false; 5707 s32 cpu; 5708 5709 for_each_cpu(cpu, this_scx->cpus_to_kick) { 5710 should_wait |= kick_one_cpu(cpu, this_rq, pseqs); 5711 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick); 5712 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); 5713 } 5714 5715 for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) { 5716 kick_one_cpu_if_idle(cpu, this_rq); 5717 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle); 5718 } 5719 5720 if (!should_wait) 5721 return; 5722 5723 for_each_cpu(cpu, this_scx->cpus_to_wait) { 5724 unsigned long *wait_pnt_seq = &cpu_rq(cpu)->scx.pnt_seq; 5725 5726 if (cpu != cpu_of(this_rq)) { 5727 /* 5728 * Pairs with smp_store_release() issued by this CPU in 5729 * scx_next_task_picked() on the resched path. 5730 * 5731 * We busy-wait here to guarantee that no other task can 5732 * be scheduled on our core before the target CPU has 5733 * entered the resched path. 5734 */ 5735 while (smp_load_acquire(wait_pnt_seq) == pseqs[cpu]) 5736 cpu_relax(); 5737 } 5738 5739 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait); 5740 } 5741 } 5742 5743 /** 5744 * print_scx_info - print out sched_ext scheduler state 5745 * @log_lvl: the log level to use when printing 5746 * @p: target task 5747 * 5748 * If a sched_ext scheduler is enabled, print the name and state of the 5749 * scheduler. If @p is on sched_ext, print further information about the task. 5750 * 5751 * This function can be safely called on any task as long as the task_struct 5752 * itself is accessible. While safe, this function isn't synchronized and may 5753 * print out mixups or garbages of limited length. 5754 */ 5755 void print_scx_info(const char *log_lvl, struct task_struct *p) 5756 { 5757 enum scx_ops_enable_state state = scx_ops_enable_state(); 5758 const char *all = READ_ONCE(scx_switching_all) ? "+all" : ""; 5759 char runnable_at_buf[22] = "?"; 5760 struct sched_class *class; 5761 unsigned long runnable_at; 5762 5763 if (state == SCX_OPS_DISABLED) 5764 return; 5765 5766 /* 5767 * Carefully check if the task was running on sched_ext, and then 5768 * carefully copy the time it's been runnable, and its state. 5769 */ 5770 if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) || 5771 class != &ext_sched_class) { 5772 printk("%sSched_ext: %s (%s%s)", log_lvl, scx_ops.name, 5773 scx_ops_enable_state_str[state], all); 5774 return; 5775 } 5776 5777 if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at, 5778 sizeof(runnable_at))) 5779 scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms", 5780 jiffies_delta_msecs(runnable_at, jiffies)); 5781 5782 /* print everything onto one line to conserve console space */ 5783 printk("%sSched_ext: %s (%s%s), task: runnable_at=%s", 5784 log_lvl, scx_ops.name, scx_ops_enable_state_str[state], all, 5785 runnable_at_buf); 5786 } 5787 5788 static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr) 5789 { 5790 /* 5791 * SCX schedulers often have userspace components which are sometimes 5792 * involved in critial scheduling paths. PM operations involve freezing 5793 * userspace which can lead to scheduling misbehaviors including stalls. 5794 * Let's bypass while PM operations are in progress. 5795 */ 5796 switch (event) { 5797 case PM_HIBERNATION_PREPARE: 5798 case PM_SUSPEND_PREPARE: 5799 case PM_RESTORE_PREPARE: 5800 scx_ops_bypass(true); 5801 break; 5802 case PM_POST_HIBERNATION: 5803 case PM_POST_SUSPEND: 5804 case PM_POST_RESTORE: 5805 scx_ops_bypass(false); 5806 break; 5807 } 5808 5809 return NOTIFY_OK; 5810 } 5811 5812 static struct notifier_block scx_pm_notifier = { 5813 .notifier_call = scx_pm_handler, 5814 }; 5815 5816 void __init init_sched_ext_class(void) 5817 { 5818 s32 cpu, v; 5819 5820 /* 5821 * The following is to prevent the compiler from optimizing out the enum 5822 * definitions so that BPF scheduler implementations can use them 5823 * through the generated vmlinux.h. 5824 */ 5825 WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT | 5826 SCX_TG_ONLINE); 5827 5828 BUG_ON(rhashtable_init(&dsq_hash, &dsq_hash_params)); 5829 #ifdef CONFIG_SMP 5830 BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL)); 5831 BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL)); 5832 #endif 5833 scx_kick_cpus_pnt_seqs = 5834 __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * nr_cpu_ids, 5835 __alignof__(scx_kick_cpus_pnt_seqs[0])); 5836 BUG_ON(!scx_kick_cpus_pnt_seqs); 5837 5838 for_each_possible_cpu(cpu) { 5839 struct rq *rq = cpu_rq(cpu); 5840 5841 init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL); 5842 INIT_LIST_HEAD(&rq->scx.runnable_list); 5843 INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals); 5844 5845 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick, GFP_KERNEL)); 5846 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL)); 5847 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_preempt, GFP_KERNEL)); 5848 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_wait, GFP_KERNEL)); 5849 init_irq_work(&rq->scx.deferred_irq_work, deferred_irq_workfn); 5850 init_irq_work(&rq->scx.kick_cpus_irq_work, kick_cpus_irq_workfn); 5851 5852 if (cpu_online(cpu)) 5853 cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; 5854 } 5855 5856 register_sysrq_key('S', &sysrq_sched_ext_reset_op); 5857 register_sysrq_key('D', &sysrq_sched_ext_dump_op); 5858 INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn); 5859 } 5860 5861 5862 /******************************************************************************** 5863 * Helpers that can be called from the BPF scheduler. 5864 */ 5865 #include <linux/btf_ids.h> 5866 5867 __bpf_kfunc_start_defs(); 5868 5869 /** 5870 * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu() 5871 * @p: task_struct to select a CPU for 5872 * @prev_cpu: CPU @p was on previously 5873 * @wake_flags: %SCX_WAKE_* flags 5874 * @is_idle: out parameter indicating whether the returned CPU is idle 5875 * 5876 * Can only be called from ops.select_cpu() if the built-in CPU selection is 5877 * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set. 5878 * @p, @prev_cpu and @wake_flags match ops.select_cpu(). 5879 * 5880 * Returns the picked CPU with *@is_idle indicating whether the picked CPU is 5881 * currently idle and thus a good candidate for direct dispatching. 5882 */ 5883 __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, 5884 u64 wake_flags, bool *is_idle) 5885 { 5886 if (!static_branch_likely(&scx_builtin_idle_enabled)) { 5887 scx_ops_error("built-in idle tracking is disabled"); 5888 goto prev_cpu; 5889 } 5890 5891 if (!scx_kf_allowed(SCX_KF_SELECT_CPU)) 5892 goto prev_cpu; 5893 5894 #ifdef CONFIG_SMP 5895 return scx_select_cpu_dfl(p, prev_cpu, wake_flags, is_idle); 5896 #endif 5897 5898 prev_cpu: 5899 *is_idle = false; 5900 return prev_cpu; 5901 } 5902 5903 __bpf_kfunc_end_defs(); 5904 5905 BTF_KFUNCS_START(scx_kfunc_ids_select_cpu) 5906 BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_RCU) 5907 BTF_KFUNCS_END(scx_kfunc_ids_select_cpu) 5908 5909 static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = { 5910 .owner = THIS_MODULE, 5911 .set = &scx_kfunc_ids_select_cpu, 5912 }; 5913 5914 static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags) 5915 { 5916 if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH)) 5917 return false; 5918 5919 lockdep_assert_irqs_disabled(); 5920 5921 if (unlikely(!p)) { 5922 scx_ops_error("called with NULL task"); 5923 return false; 5924 } 5925 5926 if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) { 5927 scx_ops_error("invalid enq_flags 0x%llx", enq_flags); 5928 return false; 5929 } 5930 5931 return true; 5932 } 5933 5934 static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags) 5935 { 5936 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 5937 struct task_struct *ddsp_task; 5938 5939 ddsp_task = __this_cpu_read(direct_dispatch_task); 5940 if (ddsp_task) { 5941 mark_direct_dispatch(ddsp_task, p, dsq_id, enq_flags); 5942 return; 5943 } 5944 5945 if (unlikely(dspc->cursor >= scx_dsp_max_batch)) { 5946 scx_ops_error("dispatch buffer overflow"); 5947 return; 5948 } 5949 5950 dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){ 5951 .task = p, 5952 .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK, 5953 .dsq_id = dsq_id, 5954 .enq_flags = enq_flags, 5955 }; 5956 } 5957 5958 __bpf_kfunc_start_defs(); 5959 5960 /** 5961 * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ 5962 * @p: task_struct to dispatch 5963 * @dsq_id: DSQ to dispatch to 5964 * @slice: duration @p can run for in nsecs, 0 to keep the current value 5965 * @enq_flags: SCX_ENQ_* 5966 * 5967 * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe 5968 * to call this function spuriously. Can be called from ops.enqueue(), 5969 * ops.select_cpu(), and ops.dispatch(). 5970 * 5971 * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch 5972 * and @p must match the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be 5973 * used to target the local DSQ of a CPU other than the enqueueing one. Use 5974 * ops.select_cpu() to be on the target CPU in the first place. 5975 * 5976 * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p 5977 * will be directly dispatched to the corresponding dispatch queue after 5978 * ops.select_cpu() returns. If @p is dispatched to SCX_DSQ_LOCAL, it will be 5979 * dispatched to the local DSQ of the CPU returned by ops.select_cpu(). 5980 * @enq_flags are OR'd with the enqueue flags on the enqueue path before the 5981 * task is dispatched. 5982 * 5983 * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id 5984 * and this function can be called upto ops.dispatch_max_batch times to dispatch 5985 * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the 5986 * remaining slots. scx_bpf_consume() flushes the batch and resets the counter. 5987 * 5988 * This function doesn't have any locking restrictions and may be called under 5989 * BPF locks (in the future when BPF introduces more flexible locking). 5990 * 5991 * @p is allowed to run for @slice. The scheduling path is triggered on slice 5992 * exhaustion. If zero, the current residual slice is maintained. If 5993 * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with 5994 * scx_bpf_kick_cpu() to trigger scheduling. 5995 */ 5996 __bpf_kfunc void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, 5997 u64 enq_flags) 5998 { 5999 if (!scx_dispatch_preamble(p, enq_flags)) 6000 return; 6001 6002 if (slice) 6003 p->scx.slice = slice; 6004 else 6005 p->scx.slice = p->scx.slice ?: 1; 6006 6007 scx_dispatch_commit(p, dsq_id, enq_flags); 6008 } 6009 6010 /** 6011 * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ 6012 * @p: task_struct to dispatch 6013 * @dsq_id: DSQ to dispatch to 6014 * @slice: duration @p can run for in nsecs, 0 to keep the current value 6015 * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ 6016 * @enq_flags: SCX_ENQ_* 6017 * 6018 * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id. 6019 * Tasks queued into the priority queue are ordered by @vtime and always 6020 * consumed after the tasks in the FIFO queue. All other aspects are identical 6021 * to scx_bpf_dispatch(). 6022 * 6023 * @vtime ordering is according to time_before64() which considers wrapping. A 6024 * numerically larger vtime may indicate an earlier position in the ordering and 6025 * vice-versa. 6026 */ 6027 __bpf_kfunc void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, 6028 u64 slice, u64 vtime, u64 enq_flags) 6029 { 6030 if (!scx_dispatch_preamble(p, enq_flags)) 6031 return; 6032 6033 if (slice) 6034 p->scx.slice = slice; 6035 else 6036 p->scx.slice = p->scx.slice ?: 1; 6037 6038 p->scx.dsq_vtime = vtime; 6039 6040 scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); 6041 } 6042 6043 __bpf_kfunc_end_defs(); 6044 6045 BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch) 6046 BTF_ID_FLAGS(func, scx_bpf_dispatch, KF_RCU) 6047 BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime, KF_RCU) 6048 BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch) 6049 6050 static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { 6051 .owner = THIS_MODULE, 6052 .set = &scx_kfunc_ids_enqueue_dispatch, 6053 }; 6054 6055 static bool scx_dispatch_from_dsq(struct bpf_iter_scx_dsq_kern *kit, 6056 struct task_struct *p, u64 dsq_id, 6057 u64 enq_flags) 6058 { 6059 struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; 6060 struct rq *this_rq, *src_rq, *dst_rq, *locked_rq; 6061 bool dispatched = false; 6062 bool in_balance; 6063 unsigned long flags; 6064 6065 if (!scx_kf_allowed_if_unlocked() && !scx_kf_allowed(SCX_KF_DISPATCH)) 6066 return false; 6067 6068 /* 6069 * Can be called from either ops.dispatch() locking this_rq() or any 6070 * context where no rq lock is held. If latter, lock @p's task_rq which 6071 * we'll likely need anyway. 6072 */ 6073 src_rq = task_rq(p); 6074 6075 local_irq_save(flags); 6076 this_rq = this_rq(); 6077 in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE; 6078 6079 if (in_balance) { 6080 if (this_rq != src_rq) { 6081 raw_spin_rq_unlock(this_rq); 6082 raw_spin_rq_lock(src_rq); 6083 } 6084 } else { 6085 raw_spin_rq_lock(src_rq); 6086 } 6087 6088 locked_rq = src_rq; 6089 raw_spin_lock(&src_dsq->lock); 6090 6091 /* 6092 * Did someone else get to it? @p could have already left $src_dsq, got 6093 * re-enqueud, or be in the process of being consumed by someone else. 6094 */ 6095 if (unlikely(p->scx.dsq != src_dsq || 6096 u32_before(kit->cursor.priv, p->scx.dsq_seq) || 6097 p->scx.holding_cpu >= 0) || 6098 WARN_ON_ONCE(src_rq != task_rq(p))) { 6099 raw_spin_unlock(&src_dsq->lock); 6100 goto out; 6101 } 6102 6103 /* @p is still on $src_dsq and stable, determine the destination */ 6104 dst_dsq = find_dsq_for_dispatch(this_rq, dsq_id, p); 6105 6106 if (dst_dsq->id == SCX_DSQ_LOCAL) { 6107 dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); 6108 if (!task_can_run_on_remote_rq(p, dst_rq, true)) { 6109 dst_dsq = find_global_dsq(p); 6110 dst_rq = src_rq; 6111 } 6112 } else { 6113 /* no need to migrate if destination is a non-local DSQ */ 6114 dst_rq = src_rq; 6115 } 6116 6117 /* 6118 * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different 6119 * CPU, @p will be migrated. 6120 */ 6121 if (dst_dsq->id == SCX_DSQ_LOCAL) { 6122 /* @p is going from a non-local DSQ to a local DSQ */ 6123 if (src_rq == dst_rq) { 6124 task_unlink_from_dsq(p, src_dsq); 6125 move_local_task_to_local_dsq(p, enq_flags, 6126 src_dsq, dst_rq); 6127 raw_spin_unlock(&src_dsq->lock); 6128 } else { 6129 raw_spin_unlock(&src_dsq->lock); 6130 move_remote_task_to_local_dsq(p, enq_flags, 6131 src_rq, dst_rq); 6132 locked_rq = dst_rq; 6133 } 6134 } else { 6135 /* 6136 * @p is going from a non-local DSQ to a non-local DSQ. As 6137 * $src_dsq is already locked, do an abbreviated dequeue. 6138 */ 6139 task_unlink_from_dsq(p, src_dsq); 6140 p->scx.dsq = NULL; 6141 raw_spin_unlock(&src_dsq->lock); 6142 6143 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) 6144 p->scx.dsq_vtime = kit->vtime; 6145 dispatch_enqueue(dst_dsq, p, enq_flags); 6146 } 6147 6148 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) 6149 p->scx.slice = kit->slice; 6150 6151 dispatched = true; 6152 out: 6153 if (in_balance) { 6154 if (this_rq != locked_rq) { 6155 raw_spin_rq_unlock(locked_rq); 6156 raw_spin_rq_lock(this_rq); 6157 } 6158 } else { 6159 raw_spin_rq_unlock_irqrestore(locked_rq, flags); 6160 } 6161 6162 kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | 6163 __SCX_DSQ_ITER_HAS_VTIME); 6164 return dispatched; 6165 } 6166 6167 __bpf_kfunc_start_defs(); 6168 6169 /** 6170 * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots 6171 * 6172 * Can only be called from ops.dispatch(). 6173 */ 6174 __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(void) 6175 { 6176 if (!scx_kf_allowed(SCX_KF_DISPATCH)) 6177 return 0; 6178 6179 return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx->cursor); 6180 } 6181 6182 /** 6183 * scx_bpf_dispatch_cancel - Cancel the latest dispatch 6184 * 6185 * Cancel the latest dispatch. Can be called multiple times to cancel further 6186 * dispatches. Can only be called from ops.dispatch(). 6187 */ 6188 __bpf_kfunc void scx_bpf_dispatch_cancel(void) 6189 { 6190 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 6191 6192 if (!scx_kf_allowed(SCX_KF_DISPATCH)) 6193 return; 6194 6195 if (dspc->cursor > 0) 6196 dspc->cursor--; 6197 else 6198 scx_ops_error("dispatch buffer underflow"); 6199 } 6200 6201 /** 6202 * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ 6203 * @dsq_id: DSQ to consume 6204 * 6205 * Consume a task from the non-local DSQ identified by @dsq_id and transfer it 6206 * to the current CPU's local DSQ for execution. Can only be called from 6207 * ops.dispatch(). 6208 * 6209 * This function flushes the in-flight dispatches from scx_bpf_dispatch() before 6210 * trying to consume the specified DSQ. It may also grab rq locks and thus can't 6211 * be called under any BPF locks. 6212 * 6213 * Returns %true if a task has been consumed, %false if there isn't any task to 6214 * consume. 6215 */ 6216 __bpf_kfunc bool scx_bpf_consume(u64 dsq_id) 6217 { 6218 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx); 6219 struct scx_dispatch_q *dsq; 6220 6221 if (!scx_kf_allowed(SCX_KF_DISPATCH)) 6222 return false; 6223 6224 flush_dispatch_buf(dspc->rq); 6225 6226 dsq = find_user_dsq(dsq_id); 6227 if (unlikely(!dsq)) { 6228 scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id); 6229 return false; 6230 } 6231 6232 if (consume_dispatch_q(dspc->rq, dsq)) { 6233 /* 6234 * A successfully consumed task can be dequeued before it starts 6235 * running while the CPU is trying to migrate other dispatched 6236 * tasks. Bump nr_tasks to tell balance_scx() to retry on empty 6237 * local DSQ. 6238 */ 6239 dspc->nr_tasks++; 6240 return true; 6241 } else { 6242 return false; 6243 } 6244 } 6245 6246 /** 6247 * scx_bpf_dispatch_from_dsq_set_slice - Override slice when dispatching from DSQ 6248 * @it__iter: DSQ iterator in progress 6249 * @slice: duration the dispatched task can run for in nsecs 6250 * 6251 * Override the slice of the next task that will be dispatched from @it__iter 6252 * using scx_bpf_dispatch_from_dsq[_vtime](). If this function is not called, 6253 * the previous slice duration is kept. 6254 */ 6255 __bpf_kfunc void scx_bpf_dispatch_from_dsq_set_slice( 6256 struct bpf_iter_scx_dsq *it__iter, u64 slice) 6257 { 6258 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; 6259 6260 kit->slice = slice; 6261 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; 6262 } 6263 6264 /** 6265 * scx_bpf_dispatch_from_dsq_set_vtime - Override vtime when dispatching from DSQ 6266 * @it__iter: DSQ iterator in progress 6267 * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ 6268 * 6269 * Override the vtime of the next task that will be dispatched from @it__iter 6270 * using scx_bpf_dispatch_from_dsq_vtime(). If this function is not called, the 6271 * previous slice vtime is kept. If scx_bpf_dispatch_from_dsq() is used to 6272 * dispatch the next task, the override is ignored and cleared. 6273 */ 6274 __bpf_kfunc void scx_bpf_dispatch_from_dsq_set_vtime( 6275 struct bpf_iter_scx_dsq *it__iter, u64 vtime) 6276 { 6277 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; 6278 6279 kit->vtime = vtime; 6280 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; 6281 } 6282 6283 /** 6284 * scx_bpf_dispatch_from_dsq - Move a task from DSQ iteration to a DSQ 6285 * @it__iter: DSQ iterator in progress 6286 * @p: task to transfer 6287 * @dsq_id: DSQ to move @p to 6288 * @enq_flags: SCX_ENQ_* 6289 * 6290 * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ 6291 * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can 6292 * be the destination. 6293 * 6294 * For the transfer to be successful, @p must still be on the DSQ and have been 6295 * queued before the DSQ iteration started. This function doesn't care whether 6296 * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have 6297 * been queued before the iteration started. 6298 * 6299 * @p's slice is kept by default. Use scx_bpf_dispatch_from_dsq_set_slice() to 6300 * update. 6301 * 6302 * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq 6303 * lock (e.g. BPF timers or SYSCALL programs). 6304 * 6305 * Returns %true if @p has been consumed, %false if @p had already been consumed 6306 * or dequeued. 6307 */ 6308 __bpf_kfunc bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq *it__iter, 6309 struct task_struct *p, u64 dsq_id, 6310 u64 enq_flags) 6311 { 6312 return scx_dispatch_from_dsq((struct bpf_iter_scx_dsq_kern *)it__iter, 6313 p, dsq_id, enq_flags); 6314 } 6315 6316 /** 6317 * scx_bpf_dispatch_vtime_from_dsq - Move a task from DSQ iteration to a PRIQ DSQ 6318 * @it__iter: DSQ iterator in progress 6319 * @p: task to transfer 6320 * @dsq_id: DSQ to move @p to 6321 * @enq_flags: SCX_ENQ_* 6322 * 6323 * Transfer @p which is on the DSQ currently iterated by @it__iter to the 6324 * priority queue of the DSQ specified by @dsq_id. The destination must be a 6325 * user DSQ as only user DSQs support priority queue. 6326 * 6327 * @p's slice and vtime are kept by default. Use 6328 * scx_bpf_dispatch_from_dsq_set_slice() and 6329 * scx_bpf_dispatch_from_dsq_set_vtime() to update. 6330 * 6331 * All other aspects are identical to scx_bpf_dispatch_from_dsq(). See 6332 * scx_bpf_dispatch_vtime() for more information on @vtime. 6333 */ 6334 __bpf_kfunc bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq *it__iter, 6335 struct task_struct *p, u64 dsq_id, 6336 u64 enq_flags) 6337 { 6338 return scx_dispatch_from_dsq((struct bpf_iter_scx_dsq_kern *)it__iter, 6339 p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ); 6340 } 6341 6342 __bpf_kfunc_end_defs(); 6343 6344 BTF_KFUNCS_START(scx_kfunc_ids_dispatch) 6345 BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots) 6346 BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel) 6347 BTF_ID_FLAGS(func, scx_bpf_consume) 6348 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_slice) 6349 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_vtime) 6350 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq, KF_RCU) 6351 BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime_from_dsq, KF_RCU) 6352 BTF_KFUNCS_END(scx_kfunc_ids_dispatch) 6353 6354 static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = { 6355 .owner = THIS_MODULE, 6356 .set = &scx_kfunc_ids_dispatch, 6357 }; 6358 6359 __bpf_kfunc_start_defs(); 6360 6361 /** 6362 * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ 6363 * 6364 * Iterate over all of the tasks currently enqueued on the local DSQ of the 6365 * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of 6366 * processed tasks. Can only be called from ops.cpu_release(). 6367 */ 6368 __bpf_kfunc u32 scx_bpf_reenqueue_local(void) 6369 { 6370 LIST_HEAD(tasks); 6371 u32 nr_enqueued = 0; 6372 struct rq *rq; 6373 struct task_struct *p, *n; 6374 6375 if (!scx_kf_allowed(SCX_KF_CPU_RELEASE)) 6376 return 0; 6377 6378 rq = cpu_rq(smp_processor_id()); 6379 lockdep_assert_rq_held(rq); 6380 6381 /* 6382 * The BPF scheduler may choose to dispatch tasks back to 6383 * @rq->scx.local_dsq. Move all candidate tasks off to a private list 6384 * first to avoid processing the same tasks repeatedly. 6385 */ 6386 list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list, 6387 scx.dsq_list.node) { 6388 /* 6389 * If @p is being migrated, @p's current CPU may not agree with 6390 * its allowed CPUs and the migration_cpu_stop is about to 6391 * deactivate and re-activate @p anyway. Skip re-enqueueing. 6392 * 6393 * While racing sched property changes may also dequeue and 6394 * re-enqueue a migrating task while its current CPU and allowed 6395 * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to 6396 * the current local DSQ for running tasks and thus are not 6397 * visible to the BPF scheduler. 6398 */ 6399 if (p->migration_pending) 6400 continue; 6401 6402 dispatch_dequeue(rq, p); 6403 list_add_tail(&p->scx.dsq_list.node, &tasks); 6404 } 6405 6406 list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { 6407 list_del_init(&p->scx.dsq_list.node); 6408 do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); 6409 nr_enqueued++; 6410 } 6411 6412 return nr_enqueued; 6413 } 6414 6415 __bpf_kfunc_end_defs(); 6416 6417 BTF_KFUNCS_START(scx_kfunc_ids_cpu_release) 6418 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local) 6419 BTF_KFUNCS_END(scx_kfunc_ids_cpu_release) 6420 6421 static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = { 6422 .owner = THIS_MODULE, 6423 .set = &scx_kfunc_ids_cpu_release, 6424 }; 6425 6426 __bpf_kfunc_start_defs(); 6427 6428 /** 6429 * scx_bpf_create_dsq - Create a custom DSQ 6430 * @dsq_id: DSQ to create 6431 * @node: NUMA node to allocate from 6432 * 6433 * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable 6434 * scx callback, and any BPF_PROG_TYPE_SYSCALL prog. 6435 */ 6436 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) 6437 { 6438 if (unlikely(node >= (int)nr_node_ids || 6439 (node < 0 && node != NUMA_NO_NODE))) 6440 return -EINVAL; 6441 return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node)); 6442 } 6443 6444 __bpf_kfunc_end_defs(); 6445 6446 BTF_KFUNCS_START(scx_kfunc_ids_unlocked) 6447 BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_SLEEPABLE) 6448 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq, KF_RCU) 6449 BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime_from_dsq, KF_RCU) 6450 BTF_KFUNCS_END(scx_kfunc_ids_unlocked) 6451 6452 static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = { 6453 .owner = THIS_MODULE, 6454 .set = &scx_kfunc_ids_unlocked, 6455 }; 6456 6457 __bpf_kfunc_start_defs(); 6458 6459 /** 6460 * scx_bpf_kick_cpu - Trigger reschedule on a CPU 6461 * @cpu: cpu to kick 6462 * @flags: %SCX_KICK_* flags 6463 * 6464 * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or 6465 * trigger rescheduling on a busy CPU. This can be called from any online 6466 * scx_ops operation and the actual kicking is performed asynchronously through 6467 * an irq work. 6468 */ 6469 __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags) 6470 { 6471 struct rq *this_rq; 6472 unsigned long irq_flags; 6473 6474 if (!ops_cpu_valid(cpu, NULL)) 6475 return; 6476 6477 local_irq_save(irq_flags); 6478 6479 this_rq = this_rq(); 6480 6481 /* 6482 * While bypassing for PM ops, IRQ handling may not be online which can 6483 * lead to irq_work_queue() malfunction such as infinite busy wait for 6484 * IRQ status update. Suppress kicking. 6485 */ 6486 if (scx_rq_bypassing(this_rq)) 6487 goto out; 6488 6489 /* 6490 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting 6491 * rq locks. We can probably be smarter and avoid bouncing if called 6492 * from ops which don't hold a rq lock. 6493 */ 6494 if (flags & SCX_KICK_IDLE) { 6495 struct rq *target_rq = cpu_rq(cpu); 6496 6497 if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) 6498 scx_ops_error("PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); 6499 6500 if (raw_spin_rq_trylock(target_rq)) { 6501 if (can_skip_idle_kick(target_rq)) { 6502 raw_spin_rq_unlock(target_rq); 6503 goto out; 6504 } 6505 raw_spin_rq_unlock(target_rq); 6506 } 6507 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle); 6508 } else { 6509 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick); 6510 6511 if (flags & SCX_KICK_PREEMPT) 6512 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt); 6513 if (flags & SCX_KICK_WAIT) 6514 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait); 6515 } 6516 6517 irq_work_queue(&this_rq->scx.kick_cpus_irq_work); 6518 out: 6519 local_irq_restore(irq_flags); 6520 } 6521 6522 /** 6523 * scx_bpf_dsq_nr_queued - Return the number of queued tasks 6524 * @dsq_id: id of the DSQ 6525 * 6526 * Return the number of tasks in the DSQ matching @dsq_id. If not found, 6527 * -%ENOENT is returned. 6528 */ 6529 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id) 6530 { 6531 struct scx_dispatch_q *dsq; 6532 s32 ret; 6533 6534 preempt_disable(); 6535 6536 if (dsq_id == SCX_DSQ_LOCAL) { 6537 ret = READ_ONCE(this_rq()->scx.local_dsq.nr); 6538 goto out; 6539 } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) { 6540 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK; 6541 6542 if (ops_cpu_valid(cpu, NULL)) { 6543 ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr); 6544 goto out; 6545 } 6546 } else { 6547 dsq = find_user_dsq(dsq_id); 6548 if (dsq) { 6549 ret = READ_ONCE(dsq->nr); 6550 goto out; 6551 } 6552 } 6553 ret = -ENOENT; 6554 out: 6555 preempt_enable(); 6556 return ret; 6557 } 6558 6559 /** 6560 * scx_bpf_destroy_dsq - Destroy a custom DSQ 6561 * @dsq_id: DSQ to destroy 6562 * 6563 * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with 6564 * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is 6565 * empty and no further tasks are dispatched to it. Ignored if called on a DSQ 6566 * which doesn't exist. Can be called from any online scx_ops operations. 6567 */ 6568 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id) 6569 { 6570 destroy_dsq(dsq_id); 6571 } 6572 6573 /** 6574 * bpf_iter_scx_dsq_new - Create a DSQ iterator 6575 * @it: iterator to initialize 6576 * @dsq_id: DSQ to iterate 6577 * @flags: %SCX_DSQ_ITER_* 6578 * 6579 * Initialize BPF iterator @it which can be used with bpf_for_each() to walk 6580 * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes 6581 * tasks which are already queued when this function is invoked. 6582 */ 6583 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, 6584 u64 flags) 6585 { 6586 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 6587 6588 BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) > 6589 sizeof(struct bpf_iter_scx_dsq)); 6590 BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != 6591 __alignof__(struct bpf_iter_scx_dsq)); 6592 6593 if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) 6594 return -EINVAL; 6595 6596 kit->dsq = find_user_dsq(dsq_id); 6597 if (!kit->dsq) 6598 return -ENOENT; 6599 6600 INIT_LIST_HEAD(&kit->cursor.node); 6601 kit->cursor.flags |= SCX_DSQ_LNODE_ITER_CURSOR | flags; 6602 kit->cursor.priv = READ_ONCE(kit->dsq->seq); 6603 6604 return 0; 6605 } 6606 6607 /** 6608 * bpf_iter_scx_dsq_next - Progress a DSQ iterator 6609 * @it: iterator to progress 6610 * 6611 * Return the next task. See bpf_iter_scx_dsq_new(). 6612 */ 6613 __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) 6614 { 6615 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 6616 bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV; 6617 struct task_struct *p; 6618 unsigned long flags; 6619 6620 if (!kit->dsq) 6621 return NULL; 6622 6623 raw_spin_lock_irqsave(&kit->dsq->lock, flags); 6624 6625 if (list_empty(&kit->cursor.node)) 6626 p = NULL; 6627 else 6628 p = container_of(&kit->cursor, struct task_struct, scx.dsq_list); 6629 6630 /* 6631 * Only tasks which were queued before the iteration started are 6632 * visible. This bounds BPF iterations and guarantees that vtime never 6633 * jumps in the other direction while iterating. 6634 */ 6635 do { 6636 p = nldsq_next_task(kit->dsq, p, rev); 6637 } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq))); 6638 6639 if (p) { 6640 if (rev) 6641 list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node); 6642 else 6643 list_move(&kit->cursor.node, &p->scx.dsq_list.node); 6644 } else { 6645 list_del_init(&kit->cursor.node); 6646 } 6647 6648 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); 6649 6650 return p; 6651 } 6652 6653 /** 6654 * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator 6655 * @it: iterator to destroy 6656 * 6657 * Undo scx_iter_scx_dsq_new(). 6658 */ 6659 __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) 6660 { 6661 struct bpf_iter_scx_dsq_kern *kit = (void *)it; 6662 6663 if (!kit->dsq) 6664 return; 6665 6666 if (!list_empty(&kit->cursor.node)) { 6667 unsigned long flags; 6668 6669 raw_spin_lock_irqsave(&kit->dsq->lock, flags); 6670 list_del_init(&kit->cursor.node); 6671 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); 6672 } 6673 kit->dsq = NULL; 6674 } 6675 6676 __bpf_kfunc_end_defs(); 6677 6678 static s32 __bstr_format(u64 *data_buf, char *line_buf, size_t line_size, 6679 char *fmt, unsigned long long *data, u32 data__sz) 6680 { 6681 struct bpf_bprintf_data bprintf_data = { .get_bin_args = true }; 6682 s32 ret; 6683 6684 if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 || 6685 (data__sz && !data)) { 6686 scx_ops_error("invalid data=%p and data__sz=%u", 6687 (void *)data, data__sz); 6688 return -EINVAL; 6689 } 6690 6691 ret = copy_from_kernel_nofault(data_buf, data, data__sz); 6692 if (ret < 0) { 6693 scx_ops_error("failed to read data fields (%d)", ret); 6694 return ret; 6695 } 6696 6697 ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8, 6698 &bprintf_data); 6699 if (ret < 0) { 6700 scx_ops_error("format preparation failed (%d)", ret); 6701 return ret; 6702 } 6703 6704 ret = bstr_printf(line_buf, line_size, fmt, 6705 bprintf_data.bin_args); 6706 bpf_bprintf_cleanup(&bprintf_data); 6707 if (ret < 0) { 6708 scx_ops_error("(\"%s\", %p, %u) failed to format", 6709 fmt, data, data__sz); 6710 return ret; 6711 } 6712 6713 return ret; 6714 } 6715 6716 static s32 bstr_format(struct scx_bstr_buf *buf, 6717 char *fmt, unsigned long long *data, u32 data__sz) 6718 { 6719 return __bstr_format(buf->data, buf->line, sizeof(buf->line), 6720 fmt, data, data__sz); 6721 } 6722 6723 __bpf_kfunc_start_defs(); 6724 6725 /** 6726 * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler. 6727 * @exit_code: Exit value to pass to user space via struct scx_exit_info. 6728 * @fmt: error message format string 6729 * @data: format string parameters packaged using ___bpf_fill() macro 6730 * @data__sz: @data len, must end in '__sz' for the verifier 6731 * 6732 * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops 6733 * disabling. 6734 */ 6735 __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt, 6736 unsigned long long *data, u32 data__sz) 6737 { 6738 unsigned long flags; 6739 6740 raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); 6741 if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0) 6742 scx_ops_exit_kind(SCX_EXIT_UNREG_BPF, exit_code, "%s", 6743 scx_exit_bstr_buf.line); 6744 raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); 6745 } 6746 6747 /** 6748 * scx_bpf_error_bstr - Indicate fatal error 6749 * @fmt: error message format string 6750 * @data: format string parameters packaged using ___bpf_fill() macro 6751 * @data__sz: @data len, must end in '__sz' for the verifier 6752 * 6753 * Indicate that the BPF scheduler encountered a fatal error and initiate ops 6754 * disabling. 6755 */ 6756 __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data, 6757 u32 data__sz) 6758 { 6759 unsigned long flags; 6760 6761 raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags); 6762 if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0) 6763 scx_ops_exit_kind(SCX_EXIT_ERROR_BPF, 0, "%s", 6764 scx_exit_bstr_buf.line); 6765 raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags); 6766 } 6767 6768 /** 6769 * scx_bpf_dump - Generate extra debug dump specific to the BPF scheduler 6770 * @fmt: format string 6771 * @data: format string parameters packaged using ___bpf_fill() macro 6772 * @data__sz: @data len, must end in '__sz' for the verifier 6773 * 6774 * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and 6775 * dump_task() to generate extra debug dump specific to the BPF scheduler. 6776 * 6777 * The extra dump may be multiple lines. A single line may be split over 6778 * multiple calls. The last line is automatically terminated. 6779 */ 6780 __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, 6781 u32 data__sz) 6782 { 6783 struct scx_dump_data *dd = &scx_dump_data; 6784 struct scx_bstr_buf *buf = &dd->buf; 6785 s32 ret; 6786 6787 if (raw_smp_processor_id() != dd->cpu) { 6788 scx_ops_error("scx_bpf_dump() must only be called from ops.dump() and friends"); 6789 return; 6790 } 6791 6792 /* append the formatted string to the line buf */ 6793 ret = __bstr_format(buf->data, buf->line + dd->cursor, 6794 sizeof(buf->line) - dd->cursor, fmt, data, data__sz); 6795 if (ret < 0) { 6796 dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)", 6797 dd->prefix, fmt, data, data__sz, ret); 6798 return; 6799 } 6800 6801 dd->cursor += ret; 6802 dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line)); 6803 6804 if (!dd->cursor) 6805 return; 6806 6807 /* 6808 * If the line buf overflowed or ends in a newline, flush it into the 6809 * dump. This is to allow the caller to generate a single line over 6810 * multiple calls. As ops_dump_flush() can also handle multiple lines in 6811 * the line buf, the only case which can lead to an unexpected 6812 * truncation is when the caller keeps generating newlines in the middle 6813 * instead of the end consecutively. Don't do that. 6814 */ 6815 if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n') 6816 ops_dump_flush(); 6817 } 6818 6819 /** 6820 * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU 6821 * @cpu: CPU of interest 6822 * 6823 * Return the maximum relative capacity of @cpu in relation to the most 6824 * performant CPU in the system. The return value is in the range [1, 6825 * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur(). 6826 */ 6827 __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu) 6828 { 6829 if (ops_cpu_valid(cpu, NULL)) 6830 return arch_scale_cpu_capacity(cpu); 6831 else 6832 return SCX_CPUPERF_ONE; 6833 } 6834 6835 /** 6836 * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU 6837 * @cpu: CPU of interest 6838 * 6839 * Return the current relative performance of @cpu in relation to its maximum. 6840 * The return value is in the range [1, %SCX_CPUPERF_ONE]. 6841 * 6842 * The current performance level of a CPU in relation to the maximum performance 6843 * available in the system can be calculated as follows: 6844 * 6845 * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE 6846 * 6847 * The result is in the range [1, %SCX_CPUPERF_ONE]. 6848 */ 6849 __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu) 6850 { 6851 if (ops_cpu_valid(cpu, NULL)) 6852 return arch_scale_freq_capacity(cpu); 6853 else 6854 return SCX_CPUPERF_ONE; 6855 } 6856 6857 /** 6858 * scx_bpf_cpuperf_set - Set the relative performance target of a CPU 6859 * @cpu: CPU of interest 6860 * @perf: target performance level [0, %SCX_CPUPERF_ONE] 6861 * @flags: %SCX_CPUPERF_* flags 6862 * 6863 * Set the target performance level of @cpu to @perf. @perf is in linear 6864 * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the 6865 * schedutil cpufreq governor chooses the target frequency. 6866 * 6867 * The actual performance level chosen, CPU grouping, and the overhead and 6868 * latency of the operations are dependent on the hardware and cpufreq driver in 6869 * use. Consult hardware and cpufreq documentation for more information. The 6870 * current performance level can be monitored using scx_bpf_cpuperf_cur(). 6871 */ 6872 __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf) 6873 { 6874 if (unlikely(perf > SCX_CPUPERF_ONE)) { 6875 scx_ops_error("Invalid cpuperf target %u for CPU %d", perf, cpu); 6876 return; 6877 } 6878 6879 if (ops_cpu_valid(cpu, NULL)) { 6880 struct rq *rq = cpu_rq(cpu); 6881 6882 rq->scx.cpuperf_target = perf; 6883 6884 rcu_read_lock_sched_notrace(); 6885 cpufreq_update_util(cpu_rq(cpu), 0); 6886 rcu_read_unlock_sched_notrace(); 6887 } 6888 } 6889 6890 /** 6891 * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs 6892 * 6893 * All valid CPU IDs in the system are smaller than the returned value. 6894 */ 6895 __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void) 6896 { 6897 return nr_cpu_ids; 6898 } 6899 6900 /** 6901 * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask 6902 */ 6903 __bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void) 6904 { 6905 return cpu_possible_mask; 6906 } 6907 6908 /** 6909 * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask 6910 */ 6911 __bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void) 6912 { 6913 return cpu_online_mask; 6914 } 6915 6916 /** 6917 * scx_bpf_put_cpumask - Release a possible/online cpumask 6918 * @cpumask: cpumask to release 6919 */ 6920 __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) 6921 { 6922 /* 6923 * Empty function body because we aren't actually acquiring or releasing 6924 * a reference to a global cpumask, which is read-only in the caller and 6925 * is never released. The acquire / release semantics here are just used 6926 * to make the cpumask is a trusted pointer in the caller. 6927 */ 6928 } 6929 6930 /** 6931 * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking 6932 * per-CPU cpumask. 6933 * 6934 * Returns NULL if idle tracking is not enabled, or running on a UP kernel. 6935 */ 6936 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(void) 6937 { 6938 if (!static_branch_likely(&scx_builtin_idle_enabled)) { 6939 scx_ops_error("built-in idle tracking is disabled"); 6940 return cpu_none_mask; 6941 } 6942 6943 #ifdef CONFIG_SMP 6944 return idle_masks.cpu; 6945 #else 6946 return cpu_none_mask; 6947 #endif 6948 } 6949 6950 /** 6951 * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking, 6952 * per-physical-core cpumask. Can be used to determine if an entire physical 6953 * core is free. 6954 * 6955 * Returns NULL if idle tracking is not enabled, or running on a UP kernel. 6956 */ 6957 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(void) 6958 { 6959 if (!static_branch_likely(&scx_builtin_idle_enabled)) { 6960 scx_ops_error("built-in idle tracking is disabled"); 6961 return cpu_none_mask; 6962 } 6963 6964 #ifdef CONFIG_SMP 6965 if (sched_smt_active()) 6966 return idle_masks.smt; 6967 else 6968 return idle_masks.cpu; 6969 #else 6970 return cpu_none_mask; 6971 #endif 6972 } 6973 6974 /** 6975 * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to 6976 * either the percpu, or SMT idle-tracking cpumask. 6977 */ 6978 __bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask) 6979 { 6980 /* 6981 * Empty function body because we aren't actually acquiring or releasing 6982 * a reference to a global idle cpumask, which is read-only in the 6983 * caller and is never released. The acquire / release semantics here 6984 * are just used to make the cpumask a trusted pointer in the caller. 6985 */ 6986 } 6987 6988 /** 6989 * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state 6990 * @cpu: cpu to test and clear idle for 6991 * 6992 * Returns %true if @cpu was idle and its idle state was successfully cleared. 6993 * %false otherwise. 6994 * 6995 * Unavailable if ops.update_idle() is implemented and 6996 * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. 6997 */ 6998 __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) 6999 { 7000 if (!static_branch_likely(&scx_builtin_idle_enabled)) { 7001 scx_ops_error("built-in idle tracking is disabled"); 7002 return false; 7003 } 7004 7005 if (ops_cpu_valid(cpu, NULL)) 7006 return test_and_clear_cpu_idle(cpu); 7007 else 7008 return false; 7009 } 7010 7011 /** 7012 * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu 7013 * @cpus_allowed: Allowed cpumask 7014 * @flags: %SCX_PICK_IDLE_CPU_* flags 7015 * 7016 * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu 7017 * number on success. -%EBUSY if no matching cpu was found. 7018 * 7019 * Idle CPU tracking may race against CPU scheduling state transitions. For 7020 * example, this function may return -%EBUSY as CPUs are transitioning into the 7021 * idle state. If the caller then assumes that there will be dispatch events on 7022 * the CPUs as they were all busy, the scheduler may end up stalling with CPUs 7023 * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and 7024 * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch 7025 * event in the near future. 7026 * 7027 * Unavailable if ops.update_idle() is implemented and 7028 * %SCX_OPS_KEEP_BUILTIN_IDLE is not set. 7029 */ 7030 __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed, 7031 u64 flags) 7032 { 7033 if (!static_branch_likely(&scx_builtin_idle_enabled)) { 7034 scx_ops_error("built-in idle tracking is disabled"); 7035 return -EBUSY; 7036 } 7037 7038 return scx_pick_idle_cpu(cpus_allowed, flags); 7039 } 7040 7041 /** 7042 * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU 7043 * @cpus_allowed: Allowed cpumask 7044 * @flags: %SCX_PICK_IDLE_CPU_* flags 7045 * 7046 * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any 7047 * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu 7048 * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is 7049 * empty. 7050 * 7051 * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not 7052 * set, this function can't tell which CPUs are idle and will always pick any 7053 * CPU. 7054 */ 7055 __bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed, 7056 u64 flags) 7057 { 7058 s32 cpu; 7059 7060 if (static_branch_likely(&scx_builtin_idle_enabled)) { 7061 cpu = scx_pick_idle_cpu(cpus_allowed, flags); 7062 if (cpu >= 0) 7063 return cpu; 7064 } 7065 7066 cpu = cpumask_any_distribute(cpus_allowed); 7067 if (cpu < nr_cpu_ids) 7068 return cpu; 7069 else 7070 return -EBUSY; 7071 } 7072 7073 /** 7074 * scx_bpf_task_running - Is task currently running? 7075 * @p: task of interest 7076 */ 7077 __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) 7078 { 7079 return task_rq(p)->curr == p; 7080 } 7081 7082 /** 7083 * scx_bpf_task_cpu - CPU a task is currently associated with 7084 * @p: task of interest 7085 */ 7086 __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p) 7087 { 7088 return task_cpu(p); 7089 } 7090 7091 /** 7092 * scx_bpf_cpu_rq - Fetch the rq of a CPU 7093 * @cpu: CPU of the rq 7094 */ 7095 __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu) 7096 { 7097 if (!ops_cpu_valid(cpu, NULL)) 7098 return NULL; 7099 7100 return cpu_rq(cpu); 7101 } 7102 7103 /** 7104 * scx_bpf_task_cgroup - Return the sched cgroup of a task 7105 * @p: task of interest 7106 * 7107 * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with 7108 * from the scheduler's POV. SCX operations should use this function to 7109 * determine @p's current cgroup as, unlike following @p->cgroups, 7110 * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all 7111 * rq-locked operations. Can be called on the parameter tasks of rq-locked 7112 * operations. The restriction guarantees that @p's rq is locked by the caller. 7113 */ 7114 #ifdef CONFIG_CGROUP_SCHED 7115 __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) 7116 { 7117 struct task_group *tg = p->sched_task_group; 7118 struct cgroup *cgrp = &cgrp_dfl_root.cgrp; 7119 7120 if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p)) 7121 goto out; 7122 7123 /* 7124 * A task_group may either be a cgroup or an autogroup. In the latter 7125 * case, @tg->css.cgroup is %NULL. A task_group can't become the other 7126 * kind once created. 7127 */ 7128 if (tg && tg->css.cgroup) 7129 cgrp = tg->css.cgroup; 7130 else 7131 cgrp = &cgrp_dfl_root.cgrp; 7132 out: 7133 cgroup_get(cgrp); 7134 return cgrp; 7135 } 7136 #endif 7137 7138 __bpf_kfunc_end_defs(); 7139 7140 BTF_KFUNCS_START(scx_kfunc_ids_any) 7141 BTF_ID_FLAGS(func, scx_bpf_kick_cpu) 7142 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued) 7143 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq) 7144 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_ITER_NEW | KF_RCU_PROTECTED) 7145 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL) 7146 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY) 7147 BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_TRUSTED_ARGS) 7148 BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS) 7149 BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_TRUSTED_ARGS) 7150 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap) 7151 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur) 7152 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set) 7153 BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids) 7154 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE) 7155 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE) 7156 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE) 7157 BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE) 7158 BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE) 7159 BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE) 7160 BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle) 7161 BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_RCU) 7162 BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_RCU) 7163 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU) 7164 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU) 7165 BTF_ID_FLAGS(func, scx_bpf_cpu_rq) 7166 #ifdef CONFIG_CGROUP_SCHED 7167 BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE) 7168 #endif 7169 BTF_KFUNCS_END(scx_kfunc_ids_any) 7170 7171 static const struct btf_kfunc_id_set scx_kfunc_set_any = { 7172 .owner = THIS_MODULE, 7173 .set = &scx_kfunc_ids_any, 7174 }; 7175 7176 static int __init scx_init(void) 7177 { 7178 int ret; 7179 7180 /* 7181 * kfunc registration can't be done from init_sched_ext_class() as 7182 * register_btf_kfunc_id_set() needs most of the system to be up. 7183 * 7184 * Some kfuncs are context-sensitive and can only be called from 7185 * specific SCX ops. They are grouped into BTF sets accordingly. 7186 * Unfortunately, BPF currently doesn't have a way of enforcing such 7187 * restrictions. Eventually, the verifier should be able to enforce 7188 * them. For now, register them the same and make each kfunc explicitly 7189 * check using scx_kf_allowed(). 7190 */ 7191 if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7192 &scx_kfunc_set_select_cpu)) || 7193 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7194 &scx_kfunc_set_enqueue_dispatch)) || 7195 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7196 &scx_kfunc_set_dispatch)) || 7197 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7198 &scx_kfunc_set_cpu_release)) || 7199 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7200 &scx_kfunc_set_unlocked)) || 7201 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, 7202 &scx_kfunc_set_unlocked)) || 7203 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, 7204 &scx_kfunc_set_any)) || 7205 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, 7206 &scx_kfunc_set_any)) || 7207 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, 7208 &scx_kfunc_set_any))) { 7209 pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret); 7210 return ret; 7211 } 7212 7213 ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops); 7214 if (ret) { 7215 pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret); 7216 return ret; 7217 } 7218 7219 ret = register_pm_notifier(&scx_pm_notifier); 7220 if (ret) { 7221 pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret); 7222 return ret; 7223 } 7224 7225 scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj); 7226 if (!scx_kset) { 7227 pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n"); 7228 return -ENOMEM; 7229 } 7230 7231 ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group); 7232 if (ret < 0) { 7233 pr_err("sched_ext: Failed to add global attributes\n"); 7234 return ret; 7235 } 7236 7237 return 0; 7238 } 7239 __initcall(scx_init); 7240