1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
4 *
5 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
6 * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
7 * Copyright (c) 2022 David Vernet <dvernet@meta.com>
8 */
9 #define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))
10
11 enum scx_consts {
12 SCX_DSP_DFL_MAX_BATCH = 32,
13 SCX_DSP_MAX_LOOPS = 32,
14 SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
15
16 SCX_EXIT_BT_LEN = 64,
17 SCX_EXIT_MSG_LEN = 1024,
18 SCX_EXIT_DUMP_DFL_LEN = 32768,
19
20 SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE,
21
22 /*
23 * Iterating all tasks may take a while. Periodically drop
24 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls.
25 */
26 SCX_OPS_TASK_ITER_BATCH = 32,
27 };
28
29 enum scx_exit_kind {
30 SCX_EXIT_NONE,
31 SCX_EXIT_DONE,
32
33 SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */
34 SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */
35 SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */
36 SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */
37
38 SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */
39 SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */
40 SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */
41 };
42
43 /*
44 * An exit code can be specified when exiting with scx_bpf_exit() or
45 * scx_ops_exit(), corresponding to exit_kind UNREG_BPF and UNREG_KERN
46 * respectively. The codes are 64bit of the format:
47 *
48 * Bits: [63 .. 48 47 .. 32 31 .. 0]
49 * [ SYS ACT ] [ SYS RSN ] [ USR ]
50 *
51 * SYS ACT: System-defined exit actions
52 * SYS RSN: System-defined exit reasons
53 * USR : User-defined exit codes and reasons
54 *
55 * Using the above, users may communicate intention and context by ORing system
56 * actions and/or system reasons with a user-defined exit code.
57 */
58 enum scx_exit_code {
59 /* Reasons */
60 SCX_ECODE_RSN_HOTPLUG = 1LLU << 32,
61
62 /* Actions */
63 SCX_ECODE_ACT_RESTART = 1LLU << 48,
64 };
65
66 /*
67 * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is
68 * being disabled.
69 */
70 struct scx_exit_info {
71 /* %SCX_EXIT_* - broad category of the exit reason */
72 enum scx_exit_kind kind;
73
74 /* exit code if gracefully exiting */
75 s64 exit_code;
76
77 /* textual representation of the above */
78 const char *reason;
79
80 /* backtrace if exiting due to an error */
81 unsigned long *bt;
82 u32 bt_len;
83
84 /* informational message */
85 char *msg;
86
87 /* debug dump */
88 char *dump;
89 };
90
91 /* sched_ext_ops.flags */
92 enum scx_ops_flags {
93 /*
94 * Keep built-in idle tracking even if ops.update_idle() is implemented.
95 */
96 SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0,
97
98 /*
99 * By default, if there are no other task to run on the CPU, ext core
100 * keeps running the current task even after its slice expires. If this
101 * flag is specified, such tasks are passed to ops.enqueue() with
102 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.
103 */
104 SCX_OPS_ENQ_LAST = 1LLU << 1,
105
106 /*
107 * An exiting task may schedule after PF_EXITING is set. In such cases,
108 * bpf_task_from_pid() may not be able to find the task and if the BPF
109 * scheduler depends on pid lookup for dispatching, the task will be
110 * lost leading to various issues including RCU grace period stalls.
111 *
112 * To mask this problem, by default, unhashed tasks are automatically
113 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't
114 * depend on pid lookups and wants to handle these tasks directly, the
115 * following flag can be used.
116 */
117 SCX_OPS_ENQ_EXITING = 1LLU << 2,
118
119 /*
120 * If set, only tasks with policy set to SCHED_EXT are attached to
121 * sched_ext. If clear, SCHED_NORMAL tasks are also included.
122 */
123 SCX_OPS_SWITCH_PARTIAL = 1LLU << 3,
124
125 /*
126 * A migration disabled task can only execute on its current CPU. By
127 * default, such tasks are automatically put on the CPU's local DSQ with
128 * the default slice on enqueue. If this ops flag is set, they also go
129 * through ops.enqueue().
130 *
131 * A migration disabled task never invokes ops.select_cpu() as it can
132 * only select the current CPU. Also, p->cpus_ptr will only contain its
133 * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr
134 * and thus may disagree with cpumask_weight(p->cpus_ptr).
135 */
136 SCX_OPS_ENQ_MIGRATION_DISABLED = 1LLU << 4,
137
138 /*
139 * CPU cgroup support flags
140 */
141 SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16, /* cpu.weight */
142
143 SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE |
144 SCX_OPS_ENQ_LAST |
145 SCX_OPS_ENQ_EXITING |
146 SCX_OPS_ENQ_MIGRATION_DISABLED |
147 SCX_OPS_SWITCH_PARTIAL |
148 SCX_OPS_HAS_CGROUP_WEIGHT,
149 };
150
151 /* argument container for ops.init_task() */
152 struct scx_init_task_args {
153 /*
154 * Set if ops.init_task() is being invoked on the fork path, as opposed
155 * to the scheduler transition path.
156 */
157 bool fork;
158 #ifdef CONFIG_EXT_GROUP_SCHED
159 /* the cgroup the task is joining */
160 struct cgroup *cgroup;
161 #endif
162 };
163
164 /* argument container for ops.exit_task() */
165 struct scx_exit_task_args {
166 /* Whether the task exited before running on sched_ext. */
167 bool cancelled;
168 };
169
170 /* argument container for ops->cgroup_init() */
171 struct scx_cgroup_init_args {
172 /* the weight of the cgroup [1..10000] */
173 u32 weight;
174 };
175
176 enum scx_cpu_preempt_reason {
177 /* next task is being scheduled by &sched_class_rt */
178 SCX_CPU_PREEMPT_RT,
179 /* next task is being scheduled by &sched_class_dl */
180 SCX_CPU_PREEMPT_DL,
181 /* next task is being scheduled by &sched_class_stop */
182 SCX_CPU_PREEMPT_STOP,
183 /* unknown reason for SCX being preempted */
184 SCX_CPU_PREEMPT_UNKNOWN,
185 };
186
187 /*
188 * Argument container for ops->cpu_acquire(). Currently empty, but may be
189 * expanded in the future.
190 */
191 struct scx_cpu_acquire_args {};
192
193 /* argument container for ops->cpu_release() */
194 struct scx_cpu_release_args {
195 /* the reason the CPU was preempted */
196 enum scx_cpu_preempt_reason reason;
197
198 /* the task that's going to be scheduled on the CPU */
199 struct task_struct *task;
200 };
201
202 /*
203 * Informational context provided to dump operations.
204 */
205 struct scx_dump_ctx {
206 enum scx_exit_kind kind;
207 s64 exit_code;
208 const char *reason;
209 u64 at_ns;
210 u64 at_jiffies;
211 };
212
213 /**
214 * struct sched_ext_ops - Operation table for BPF scheduler implementation
215 *
216 * A BPF scheduler can implement an arbitrary scheduling policy by
217 * implementing and loading operations in this table. Note that a userland
218 * scheduling policy can also be implemented using the BPF scheduler
219 * as a shim layer.
220 */
221 struct sched_ext_ops {
222 /**
223 * @select_cpu: Pick the target CPU for a task which is being woken up
224 * @p: task being woken up
225 * @prev_cpu: the cpu @p was on before sleeping
226 * @wake_flags: SCX_WAKE_*
227 *
228 * Decision made here isn't final. @p may be moved to any CPU while it
229 * is getting dispatched for execution later. However, as @p is not on
230 * the rq at this point, getting the eventual execution CPU right here
231 * saves a small bit of overhead down the line.
232 *
233 * If an idle CPU is returned, the CPU is kicked and will try to
234 * dispatch. While an explicit custom mechanism can be added,
235 * select_cpu() serves as the default way to wake up idle CPUs.
236 *
237 * @p may be inserted into a DSQ directly by calling
238 * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped.
239 * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ
240 * of the CPU returned by this operation.
241 *
242 * Note that select_cpu() is never called for tasks that can only run
243 * on a single CPU or tasks with migration disabled, as they don't have
244 * the option to select a different CPU. See select_task_rq() for
245 * details.
246 */
247 s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags);
248
249 /**
250 * @enqueue: Enqueue a task on the BPF scheduler
251 * @p: task being enqueued
252 * @enq_flags: %SCX_ENQ_*
253 *
254 * @p is ready to run. Insert directly into a DSQ by calling
255 * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly
256 * inserted, the bpf scheduler owns @p and if it fails to dispatch @p,
257 * the task will stall.
258 *
259 * If @p was inserted into a DSQ from ops.select_cpu(), this callback is
260 * skipped.
261 */
262 void (*enqueue)(struct task_struct *p, u64 enq_flags);
263
264 /**
265 * @dequeue: Remove a task from the BPF scheduler
266 * @p: task being dequeued
267 * @deq_flags: %SCX_DEQ_*
268 *
269 * Remove @p from the BPF scheduler. This is usually called to isolate
270 * the task while updating its scheduling properties (e.g. priority).
271 *
272 * The ext core keeps track of whether the BPF side owns a given task or
273 * not and can gracefully ignore spurious dispatches from BPF side,
274 * which makes it safe to not implement this method. However, depending
275 * on the scheduling logic, this can lead to confusing behaviors - e.g.
276 * scheduling position not being updated across a priority change.
277 */
278 void (*dequeue)(struct task_struct *p, u64 deq_flags);
279
280 /**
281 * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs
282 * @cpu: CPU to dispatch tasks for
283 * @prev: previous task being switched out
284 *
285 * Called when a CPU's local dsq is empty. The operation should dispatch
286 * one or more tasks from the BPF scheduler into the DSQs using
287 * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ
288 * using scx_bpf_dsq_move_to_local().
289 *
290 * The maximum number of times scx_bpf_dsq_insert() can be called
291 * without an intervening scx_bpf_dsq_move_to_local() is specified by
292 * ops.dispatch_max_batch. See the comments on top of the two functions
293 * for more details.
294 *
295 * When not %NULL, @prev is an SCX task with its slice depleted. If
296 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in
297 * @prev->scx.flags, it is not enqueued yet and will be enqueued after
298 * ops.dispatch() returns. To keep executing @prev, return without
299 * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST.
300 */
301 void (*dispatch)(s32 cpu, struct task_struct *prev);
302
303 /**
304 * @tick: Periodic tick
305 * @p: task running currently
306 *
307 * This operation is called every 1/HZ seconds on CPUs which are
308 * executing an SCX task. Setting @p->scx.slice to 0 will trigger an
309 * immediate dispatch cycle on the CPU.
310 */
311 void (*tick)(struct task_struct *p);
312
313 /**
314 * @runnable: A task is becoming runnable on its associated CPU
315 * @p: task becoming runnable
316 * @enq_flags: %SCX_ENQ_*
317 *
318 * This and the following three functions can be used to track a task's
319 * execution state transitions. A task becomes ->runnable() on a CPU,
320 * and then goes through one or more ->running() and ->stopping() pairs
321 * as it runs on the CPU, and eventually becomes ->quiescent() when it's
322 * done running on the CPU.
323 *
324 * @p is becoming runnable on the CPU because it's
325 *
326 * - waking up (%SCX_ENQ_WAKEUP)
327 * - being moved from another CPU
328 * - being restored after temporarily taken off the queue for an
329 * attribute change.
330 *
331 * This and ->enqueue() are related but not coupled. This operation
332 * notifies @p's state transition and may not be followed by ->enqueue()
333 * e.g. when @p is being dispatched to a remote CPU, or when @p is
334 * being enqueued on a CPU experiencing a hotplug event. Likewise, a
335 * task may be ->enqueue()'d without being preceded by this operation
336 * e.g. after exhausting its slice.
337 */
338 void (*runnable)(struct task_struct *p, u64 enq_flags);
339
340 /**
341 * @running: A task is starting to run on its associated CPU
342 * @p: task starting to run
343 *
344 * See ->runnable() for explanation on the task state notifiers.
345 */
346 void (*running)(struct task_struct *p);
347
348 /**
349 * @stopping: A task is stopping execution
350 * @p: task stopping to run
351 * @runnable: is task @p still runnable?
352 *
353 * See ->runnable() for explanation on the task state notifiers. If
354 * !@runnable, ->quiescent() will be invoked after this operation
355 * returns.
356 */
357 void (*stopping)(struct task_struct *p, bool runnable);
358
359 /**
360 * @quiescent: A task is becoming not runnable on its associated CPU
361 * @p: task becoming not runnable
362 * @deq_flags: %SCX_DEQ_*
363 *
364 * See ->runnable() for explanation on the task state notifiers.
365 *
366 * @p is becoming quiescent on the CPU because it's
367 *
368 * - sleeping (%SCX_DEQ_SLEEP)
369 * - being moved to another CPU
370 * - being temporarily taken off the queue for an attribute change
371 * (%SCX_DEQ_SAVE)
372 *
373 * This and ->dequeue() are related but not coupled. This operation
374 * notifies @p's state transition and may not be preceded by ->dequeue()
375 * e.g. when @p is being dispatched to a remote CPU.
376 */
377 void (*quiescent)(struct task_struct *p, u64 deq_flags);
378
379 /**
380 * @yield: Yield CPU
381 * @from: yielding task
382 * @to: optional yield target task
383 *
384 * If @to is NULL, @from is yielding the CPU to other runnable tasks.
385 * The BPF scheduler should ensure that other available tasks are
386 * dispatched before the yielding task. Return value is ignored in this
387 * case.
388 *
389 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf
390 * scheduler can implement the request, return %true; otherwise, %false.
391 */
392 bool (*yield)(struct task_struct *from, struct task_struct *to);
393
394 /**
395 * @core_sched_before: Task ordering for core-sched
396 * @a: task A
397 * @b: task B
398 *
399 * Used by core-sched to determine the ordering between two tasks. See
400 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on
401 * core-sched.
402 *
403 * Both @a and @b are runnable and may or may not currently be queued on
404 * the BPF scheduler. Should return %true if @a should run before @b.
405 * %false if there's no required ordering or @b should run before @a.
406 *
407 * If not specified, the default is ordering them according to when they
408 * became runnable.
409 */
410 bool (*core_sched_before)(struct task_struct *a, struct task_struct *b);
411
412 /**
413 * @set_weight: Set task weight
414 * @p: task to set weight for
415 * @weight: new weight [1..10000]
416 *
417 * Update @p's weight to @weight.
418 */
419 void (*set_weight)(struct task_struct *p, u32 weight);
420
421 /**
422 * @set_cpumask: Set CPU affinity
423 * @p: task to set CPU affinity for
424 * @cpumask: cpumask of cpus that @p can run on
425 *
426 * Update @p's CPU affinity to @cpumask.
427 */
428 void (*set_cpumask)(struct task_struct *p,
429 const struct cpumask *cpumask);
430
431 /**
432 * @update_idle: Update the idle state of a CPU
433 * @cpu: CPU to update the idle state for
434 * @idle: whether entering or exiting the idle state
435 *
436 * This operation is called when @rq's CPU goes or leaves the idle
437 * state. By default, implementing this operation disables the built-in
438 * idle CPU tracking and the following helpers become unavailable:
439 *
440 * - scx_bpf_select_cpu_dfl()
441 * - scx_bpf_test_and_clear_cpu_idle()
442 * - scx_bpf_pick_idle_cpu()
443 *
444 * The user also must implement ops.select_cpu() as the default
445 * implementation relies on scx_bpf_select_cpu_dfl().
446 *
447 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
448 * tracking.
449 */
450 void (*update_idle)(s32 cpu, bool idle);
451
452 /**
453 * @cpu_acquire: A CPU is becoming available to the BPF scheduler
454 * @cpu: The CPU being acquired by the BPF scheduler.
455 * @args: Acquire arguments, see the struct definition.
456 *
457 * A CPU that was previously released from the BPF scheduler is now once
458 * again under its control.
459 */
460 void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args);
461
462 /**
463 * @cpu_release: A CPU is taken away from the BPF scheduler
464 * @cpu: The CPU being released by the BPF scheduler.
465 * @args: Release arguments, see the struct definition.
466 *
467 * The specified CPU is no longer under the control of the BPF
468 * scheduler. This could be because it was preempted by a higher
469 * priority sched_class, though there may be other reasons as well. The
470 * caller should consult @args->reason to determine the cause.
471 */
472 void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args);
473
474 /**
475 * @init_task: Initialize a task to run in a BPF scheduler
476 * @p: task to initialize for BPF scheduling
477 * @args: init arguments, see the struct definition
478 *
479 * Either we're loading a BPF scheduler or a new task is being forked.
480 * Initialize @p for BPF scheduling. This operation may block and can
481 * be used for allocations, and is called exactly once for a task.
482 *
483 * Return 0 for success, -errno for failure. An error return while
484 * loading will abort loading of the BPF scheduler. During a fork, it
485 * will abort that specific fork.
486 */
487 s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args);
488
489 /**
490 * @exit_task: Exit a previously-running task from the system
491 * @p: task to exit
492 * @args: exit arguments, see the struct definition
493 *
494 * @p is exiting or the BPF scheduler is being unloaded. Perform any
495 * necessary cleanup for @p.
496 */
497 void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args);
498
499 /**
500 * @enable: Enable BPF scheduling for a task
501 * @p: task to enable BPF scheduling for
502 *
503 * Enable @p for BPF scheduling. enable() is called on @p any time it
504 * enters SCX, and is always paired with a matching disable().
505 */
506 void (*enable)(struct task_struct *p);
507
508 /**
509 * @disable: Disable BPF scheduling for a task
510 * @p: task to disable BPF scheduling for
511 *
512 * @p is exiting, leaving SCX or the BPF scheduler is being unloaded.
513 * Disable BPF scheduling for @p. A disable() call is always matched
514 * with a prior enable() call.
515 */
516 void (*disable)(struct task_struct *p);
517
518 /**
519 * @dump: Dump BPF scheduler state on error
520 * @ctx: debug dump context
521 *
522 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump.
523 */
524 void (*dump)(struct scx_dump_ctx *ctx);
525
526 /**
527 * @dump_cpu: Dump BPF scheduler state for a CPU on error
528 * @ctx: debug dump context
529 * @cpu: CPU to generate debug dump for
530 * @idle: @cpu is currently idle without any runnable tasks
531 *
532 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for
533 * @cpu. If @idle is %true and this operation doesn't produce any
534 * output, @cpu is skipped for dump.
535 */
536 void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle);
537
538 /**
539 * @dump_task: Dump BPF scheduler state for a runnable task on error
540 * @ctx: debug dump context
541 * @p: runnable task to generate debug dump for
542 *
543 * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for
544 * @p.
545 */
546 void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p);
547
548 #ifdef CONFIG_EXT_GROUP_SCHED
549 /**
550 * @cgroup_init: Initialize a cgroup
551 * @cgrp: cgroup being initialized
552 * @args: init arguments, see the struct definition
553 *
554 * Either the BPF scheduler is being loaded or @cgrp created, initialize
555 * @cgrp for sched_ext. This operation may block.
556 *
557 * Return 0 for success, -errno for failure. An error return while
558 * loading will abort loading of the BPF scheduler. During cgroup
559 * creation, it will abort the specific cgroup creation.
560 */
561 s32 (*cgroup_init)(struct cgroup *cgrp,
562 struct scx_cgroup_init_args *args);
563
564 /**
565 * @cgroup_exit: Exit a cgroup
566 * @cgrp: cgroup being exited
567 *
568 * Either the BPF scheduler is being unloaded or @cgrp destroyed, exit
569 * @cgrp for sched_ext. This operation my block.
570 */
571 void (*cgroup_exit)(struct cgroup *cgrp);
572
573 /**
574 * @cgroup_prep_move: Prepare a task to be moved to a different cgroup
575 * @p: task being moved
576 * @from: cgroup @p is being moved from
577 * @to: cgroup @p is being moved to
578 *
579 * Prepare @p for move from cgroup @from to @to. This operation may
580 * block and can be used for allocations.
581 *
582 * Return 0 for success, -errno for failure. An error return aborts the
583 * migration.
584 */
585 s32 (*cgroup_prep_move)(struct task_struct *p,
586 struct cgroup *from, struct cgroup *to);
587
588 /**
589 * @cgroup_move: Commit cgroup move
590 * @p: task being moved
591 * @from: cgroup @p is being moved from
592 * @to: cgroup @p is being moved to
593 *
594 * Commit the move. @p is dequeued during this operation.
595 */
596 void (*cgroup_move)(struct task_struct *p,
597 struct cgroup *from, struct cgroup *to);
598
599 /**
600 * @cgroup_cancel_move: Cancel cgroup move
601 * @p: task whose cgroup move is being canceled
602 * @from: cgroup @p was being moved from
603 * @to: cgroup @p was being moved to
604 *
605 * @p was cgroup_prep_move()'d but failed before reaching cgroup_move().
606 * Undo the preparation.
607 */
608 void (*cgroup_cancel_move)(struct task_struct *p,
609 struct cgroup *from, struct cgroup *to);
610
611 /**
612 * @cgroup_set_weight: A cgroup's weight is being changed
613 * @cgrp: cgroup whose weight is being updated
614 * @weight: new weight [1..10000]
615 *
616 * Update @tg's weight to @weight.
617 */
618 void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight);
619 #endif /* CONFIG_EXT_GROUP_SCHED */
620
621 /*
622 * All online ops must come before ops.cpu_online().
623 */
624
625 /**
626 * @cpu_online: A CPU became online
627 * @cpu: CPU which just came up
628 *
629 * @cpu just came online. @cpu will not call ops.enqueue() or
630 * ops.dispatch(), nor run tasks associated with other CPUs beforehand.
631 */
632 void (*cpu_online)(s32 cpu);
633
634 /**
635 * @cpu_offline: A CPU is going offline
636 * @cpu: CPU which is going offline
637 *
638 * @cpu is going offline. @cpu will not call ops.enqueue() or
639 * ops.dispatch(), nor run tasks associated with other CPUs afterwards.
640 */
641 void (*cpu_offline)(s32 cpu);
642
643 /*
644 * All CPU hotplug ops must come before ops.init().
645 */
646
647 /**
648 * @init: Initialize the BPF scheduler
649 */
650 s32 (*init)(void);
651
652 /**
653 * @exit: Clean up after the BPF scheduler
654 * @info: Exit info
655 *
656 * ops.exit() is also called on ops.init() failure, which is a bit
657 * unusual. This is to allow rich reporting through @info on how
658 * ops.init() failed.
659 */
660 void (*exit)(struct scx_exit_info *info);
661
662 /**
663 * @dispatch_max_batch: Max nr of tasks that dispatch() can dispatch
664 */
665 u32 dispatch_max_batch;
666
667 /**
668 * @flags: %SCX_OPS_* flags
669 */
670 u64 flags;
671
672 /**
673 * @timeout_ms: The maximum amount of time, in milliseconds, that a
674 * runnable task should be able to wait before being scheduled. The
675 * maximum timeout may not exceed the default timeout of 30 seconds.
676 *
677 * Defaults to the maximum allowed timeout value of 30 seconds.
678 */
679 u32 timeout_ms;
680
681 /**
682 * @exit_dump_len: scx_exit_info.dump buffer length. If 0, the default
683 * value of 32768 is used.
684 */
685 u32 exit_dump_len;
686
687 /**
688 * @hotplug_seq: A sequence number that may be set by the scheduler to
689 * detect when a hotplug event has occurred during the loading process.
690 * If 0, no detection occurs. Otherwise, the scheduler will fail to
691 * load if the sequence number does not match @scx_hotplug_seq on the
692 * enable path.
693 */
694 u64 hotplug_seq;
695
696 /**
697 * @name: BPF scheduler's name
698 *
699 * Must be a non-zero valid BPF object name including only isalnum(),
700 * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the
701 * BPF scheduler is enabled.
702 */
703 char name[SCX_OPS_NAME_LEN];
704 };
705
706 enum scx_opi {
707 SCX_OPI_BEGIN = 0,
708 SCX_OPI_NORMAL_BEGIN = 0,
709 SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online),
710 SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online),
711 SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init),
712 SCX_OPI_END = SCX_OP_IDX(init),
713 };
714
715 enum scx_wake_flags {
716 /* expose select WF_* flags as enums */
717 SCX_WAKE_FORK = WF_FORK,
718 SCX_WAKE_TTWU = WF_TTWU,
719 SCX_WAKE_SYNC = WF_SYNC,
720 };
721
722 enum scx_enq_flags {
723 /* expose select ENQUEUE_* flags as enums */
724 SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP,
725 SCX_ENQ_HEAD = ENQUEUE_HEAD,
726 SCX_ENQ_CPU_SELECTED = ENQUEUE_RQ_SELECTED,
727
728 /* high 32bits are SCX specific */
729
730 /*
731 * Set the following to trigger preemption when calling
732 * scx_bpf_dsq_insert() with a local dsq as the target. The slice of the
733 * current task is cleared to zero and the CPU is kicked into the
734 * scheduling path. Implies %SCX_ENQ_HEAD.
735 */
736 SCX_ENQ_PREEMPT = 1LLU << 32,
737
738 /*
739 * The task being enqueued was previously enqueued on the current CPU's
740 * %SCX_DSQ_LOCAL, but was removed from it in a call to the
741 * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was
742 * invoked in a ->cpu_release() callback, and the task is again
743 * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the
744 * task will not be scheduled on the CPU until at least the next invocation
745 * of the ->cpu_acquire() callback.
746 */
747 SCX_ENQ_REENQ = 1LLU << 40,
748
749 /*
750 * The task being enqueued is the only task available for the cpu. By
751 * default, ext core keeps executing such tasks but when
752 * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the
753 * %SCX_ENQ_LAST flag set.
754 *
755 * The BPF scheduler is responsible for triggering a follow-up
756 * scheduling event. Otherwise, Execution may stall.
757 */
758 SCX_ENQ_LAST = 1LLU << 41,
759
760 /* high 8 bits are internal */
761 __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56,
762
763 SCX_ENQ_CLEAR_OPSS = 1LLU << 56,
764 SCX_ENQ_DSQ_PRIQ = 1LLU << 57,
765 };
766
767 enum scx_deq_flags {
768 /* expose select DEQUEUE_* flags as enums */
769 SCX_DEQ_SLEEP = DEQUEUE_SLEEP,
770
771 /* high 32bits are SCX specific */
772
773 /*
774 * The generic core-sched layer decided to execute the task even though
775 * it hasn't been dispatched yet. Dequeue from the BPF side.
776 */
777 SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32,
778 };
779
780 enum scx_pick_idle_cpu_flags {
781 SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */
782 };
783
784 enum scx_kick_flags {
785 /*
786 * Kick the target CPU if idle. Guarantees that the target CPU goes
787 * through at least one full scheduling cycle before going idle. If the
788 * target CPU can be determined to be currently not idle and going to go
789 * through a scheduling cycle before going idle, noop.
790 */
791 SCX_KICK_IDLE = 1LLU << 0,
792
793 /*
794 * Preempt the current task and execute the dispatch path. If the
795 * current task of the target CPU is an SCX task, its ->scx.slice is
796 * cleared to zero before the scheduling path is invoked so that the
797 * task expires and the dispatch path is invoked.
798 */
799 SCX_KICK_PREEMPT = 1LLU << 1,
800
801 /*
802 * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will
803 * return after the target CPU finishes picking the next task.
804 */
805 SCX_KICK_WAIT = 1LLU << 2,
806 };
807
808 enum scx_tg_flags {
809 SCX_TG_ONLINE = 1U << 0,
810 SCX_TG_INITED = 1U << 1,
811 };
812
813 enum scx_ops_enable_state {
814 SCX_OPS_ENABLING,
815 SCX_OPS_ENABLED,
816 SCX_OPS_DISABLING,
817 SCX_OPS_DISABLED,
818 };
819
820 static const char *scx_ops_enable_state_str[] = {
821 [SCX_OPS_ENABLING] = "enabling",
822 [SCX_OPS_ENABLED] = "enabled",
823 [SCX_OPS_DISABLING] = "disabling",
824 [SCX_OPS_DISABLED] = "disabled",
825 };
826
827 /*
828 * sched_ext_entity->ops_state
829 *
830 * Used to track the task ownership between the SCX core and the BPF scheduler.
831 * State transitions look as follows:
832 *
833 * NONE -> QUEUEING -> QUEUED -> DISPATCHING
834 * ^ | |
835 * | v v
836 * \-------------------------------/
837 *
838 * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call
839 * sites for explanations on the conditions being waited upon and why they are
840 * safe. Transitions out of them into NONE or QUEUED must store_release and the
841 * waiters should load_acquire.
842 *
843 * Tracking scx_ops_state enables sched_ext core to reliably determine whether
844 * any given task can be dispatched by the BPF scheduler at all times and thus
845 * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler
846 * to try to dispatch any task anytime regardless of its state as the SCX core
847 * can safely reject invalid dispatches.
848 */
849 enum scx_ops_state {
850 SCX_OPSS_NONE, /* owned by the SCX core */
851 SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */
852 SCX_OPSS_QUEUED, /* owned by the BPF scheduler */
853 SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */
854
855 /*
856 * QSEQ brands each QUEUED instance so that, when dispatch races
857 * dequeue/requeue, the dispatcher can tell whether it still has a claim
858 * on the task being dispatched.
859 *
860 * As some 32bit archs can't do 64bit store_release/load_acquire,
861 * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on
862 * 32bit machines. The dispatch race window QSEQ protects is very narrow
863 * and runs with IRQ disabled. 30 bits should be sufficient.
864 */
865 SCX_OPSS_QSEQ_SHIFT = 2,
866 };
867
868 /* Use macros to ensure that the type is unsigned long for the masks */
869 #define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1)
870 #define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK)
871
872 /*
873 * During exit, a task may schedule after losing its PIDs. When disabling the
874 * BPF scheduler, we need to be able to iterate tasks in every state to
875 * guarantee system safety. Maintain a dedicated task list which contains every
876 * task between its fork and eventual free.
877 */
878 static DEFINE_SPINLOCK(scx_tasks_lock);
879 static LIST_HEAD(scx_tasks);
880
881 /* ops enable/disable */
882 static struct kthread_worker *scx_ops_helper;
883 static DEFINE_MUTEX(scx_ops_enable_mutex);
884 DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled);
885 DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
886 static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED);
887 static unsigned long scx_in_softlockup;
888 static atomic_t scx_ops_breather_depth = ATOMIC_INIT(0);
889 static int scx_ops_bypass_depth;
890 static bool scx_ops_init_task_enabled;
891 static bool scx_switching_all;
892 DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
893
894 static struct sched_ext_ops scx_ops;
895 static bool scx_warned_zero_slice;
896
897 static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last);
898 static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting);
899 static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_migration_disabled);
900 static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);
901 static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled);
902
903 #ifdef CONFIG_SMP
904 static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_llc);
905 static DEFINE_STATIC_KEY_FALSE(scx_selcpu_topo_numa);
906 #endif
907
908 static struct static_key_false scx_has_op[SCX_OPI_END] =
909 { [0 ... SCX_OPI_END-1] = STATIC_KEY_FALSE_INIT };
910
911 static atomic_t scx_exit_kind = ATOMIC_INIT(SCX_EXIT_DONE);
912 static struct scx_exit_info *scx_exit_info;
913
914 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
915 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
916
917 /*
918 * A monotically increasing sequence number that is incremented every time a
919 * scheduler is enabled. This can be used by to check if any custom sched_ext
920 * scheduler has ever been used in the system.
921 */
922 static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0);
923
924 /*
925 * The maximum amount of time in jiffies that a task may be runnable without
926 * being scheduled on a CPU. If this timeout is exceeded, it will trigger
927 * scx_ops_error().
928 */
929 static unsigned long scx_watchdog_timeout;
930
931 /*
932 * The last time the delayed work was run. This delayed work relies on
933 * ksoftirqd being able to run to service timer interrupts, so it's possible
934 * that this work itself could get wedged. To account for this, we check that
935 * it's not stalled in the timer tick, and trigger an error if it is.
936 */
937 static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES;
938
939 static struct delayed_work scx_watchdog_work;
940
941 /* idle tracking */
942 #ifdef CONFIG_SMP
943 #ifdef CONFIG_CPUMASK_OFFSTACK
944 #define CL_ALIGNED_IF_ONSTACK
945 #else
946 #define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp
947 #endif
948
949 static struct {
950 cpumask_var_t cpu;
951 cpumask_var_t smt;
952 } idle_masks CL_ALIGNED_IF_ONSTACK;
953
954 #endif /* CONFIG_SMP */
955
956 /* for %SCX_KICK_WAIT */
957 static unsigned long __percpu *scx_kick_cpus_pnt_seqs;
958
959 /*
960 * Direct dispatch marker.
961 *
962 * Non-NULL values are used for direct dispatch from enqueue path. A valid
963 * pointer points to the task currently being enqueued. An ERR_PTR value is used
964 * to indicate that direct dispatch has already happened.
965 */
966 static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
967
968 /*
969 * Dispatch queues.
970 *
971 * The global DSQ (%SCX_DSQ_GLOBAL) is split per-node for scalability. This is
972 * to avoid live-locking in bypass mode where all tasks are dispatched to
973 * %SCX_DSQ_GLOBAL and all CPUs consume from it. If per-node split isn't
974 * sufficient, it can be further split.
975 */
976 static struct scx_dispatch_q **global_dsqs;
977
978 static const struct rhashtable_params dsq_hash_params = {
979 .key_len = sizeof_field(struct scx_dispatch_q, id),
980 .key_offset = offsetof(struct scx_dispatch_q, id),
981 .head_offset = offsetof(struct scx_dispatch_q, hash_node),
982 };
983
984 static struct rhashtable dsq_hash;
985 static LLIST_HEAD(dsqs_to_free);
986
987 /* dispatch buf */
988 struct scx_dsp_buf_ent {
989 struct task_struct *task;
990 unsigned long qseq;
991 u64 dsq_id;
992 u64 enq_flags;
993 };
994
995 static u32 scx_dsp_max_batch;
996
997 struct scx_dsp_ctx {
998 struct rq *rq;
999 u32 cursor;
1000 u32 nr_tasks;
1001 struct scx_dsp_buf_ent buf[];
1002 };
1003
1004 static struct scx_dsp_ctx __percpu *scx_dsp_ctx;
1005
1006 /* string formatting from BPF */
1007 struct scx_bstr_buf {
1008 u64 data[MAX_BPRINTF_VARARGS];
1009 char line[SCX_EXIT_MSG_LEN];
1010 };
1011
1012 static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);
1013 static struct scx_bstr_buf scx_exit_bstr_buf;
1014
1015 /* ops debug dump */
1016 struct scx_dump_data {
1017 s32 cpu;
1018 bool first;
1019 s32 cursor;
1020 struct seq_buf *s;
1021 const char *prefix;
1022 struct scx_bstr_buf buf;
1023 };
1024
1025 static struct scx_dump_data scx_dump_data = {
1026 .cpu = -1,
1027 };
1028
1029 /* /sys/kernel/sched_ext interface */
1030 static struct kset *scx_kset;
1031 static struct kobject *scx_root_kobj;
1032
1033 #define CREATE_TRACE_POINTS
1034 #include <trace/events/sched_ext.h>
1035
1036 static void process_ddsp_deferred_locals(struct rq *rq);
1037 static void scx_bpf_kick_cpu(s32 cpu, u64 flags);
1038 static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind,
1039 s64 exit_code,
1040 const char *fmt, ...);
1041
1042 #define scx_ops_error_kind(err, fmt, args...) \
1043 scx_ops_exit_kind((err), 0, fmt, ##args)
1044
1045 #define scx_ops_exit(code, fmt, args...) \
1046 scx_ops_exit_kind(SCX_EXIT_UNREG_KERN, (code), fmt, ##args)
1047
1048 #define scx_ops_error(fmt, args...) \
1049 scx_ops_error_kind(SCX_EXIT_ERROR, fmt, ##args)
1050
1051 #define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)])
1052
jiffies_delta_msecs(unsigned long at,unsigned long now)1053 static long jiffies_delta_msecs(unsigned long at, unsigned long now)
1054 {
1055 if (time_after(at, now))
1056 return jiffies_to_msecs(at - now);
1057 else
1058 return -(long)jiffies_to_msecs(now - at);
1059 }
1060
1061 /* if the highest set bit is N, return a mask with bits [N+1, 31] set */
higher_bits(u32 flags)1062 static u32 higher_bits(u32 flags)
1063 {
1064 return ~((1 << fls(flags)) - 1);
1065 }
1066
1067 /* return the mask with only the highest bit set */
highest_bit(u32 flags)1068 static u32 highest_bit(u32 flags)
1069 {
1070 int bit = fls(flags);
1071 return ((u64)1 << bit) >> 1;
1072 }
1073
u32_before(u32 a,u32 b)1074 static bool u32_before(u32 a, u32 b)
1075 {
1076 return (s32)(a - b) < 0;
1077 }
1078
find_global_dsq(struct task_struct * p)1079 static struct scx_dispatch_q *find_global_dsq(struct task_struct *p)
1080 {
1081 return global_dsqs[cpu_to_node(task_cpu(p))];
1082 }
1083
find_user_dsq(u64 dsq_id)1084 static struct scx_dispatch_q *find_user_dsq(u64 dsq_id)
1085 {
1086 return rhashtable_lookup_fast(&dsq_hash, &dsq_id, dsq_hash_params);
1087 }
1088
1089 /*
1090 * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX
1091 * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate
1092 * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check
1093 * whether it's running from an allowed context.
1094 *
1095 * @mask is constant, always inline to cull the mask calculations.
1096 */
scx_kf_allow(u32 mask)1097 static __always_inline void scx_kf_allow(u32 mask)
1098 {
1099 /* nesting is allowed only in increasing scx_kf_mask order */
1100 WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask,
1101 "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n",
1102 current->scx.kf_mask, mask);
1103 current->scx.kf_mask |= mask;
1104 barrier();
1105 }
1106
scx_kf_disallow(u32 mask)1107 static void scx_kf_disallow(u32 mask)
1108 {
1109 barrier();
1110 current->scx.kf_mask &= ~mask;
1111 }
1112
1113 #define SCX_CALL_OP(mask, op, args...) \
1114 do { \
1115 if (mask) { \
1116 scx_kf_allow(mask); \
1117 scx_ops.op(args); \
1118 scx_kf_disallow(mask); \
1119 } else { \
1120 scx_ops.op(args); \
1121 } \
1122 } while (0)
1123
1124 #define SCX_CALL_OP_RET(mask, op, args...) \
1125 ({ \
1126 __typeof__(scx_ops.op(args)) __ret; \
1127 if (mask) { \
1128 scx_kf_allow(mask); \
1129 __ret = scx_ops.op(args); \
1130 scx_kf_disallow(mask); \
1131 } else { \
1132 __ret = scx_ops.op(args); \
1133 } \
1134 __ret; \
1135 })
1136
1137 /*
1138 * Some kfuncs are allowed only on the tasks that are subjects of the
1139 * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such
1140 * restrictions, the following SCX_CALL_OP_*() variants should be used when
1141 * invoking scx_ops operations that take task arguments. These can only be used
1142 * for non-nesting operations due to the way the tasks are tracked.
1143 *
1144 * kfuncs which can only operate on such tasks can in turn use
1145 * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on
1146 * the specific task.
1147 */
1148 #define SCX_CALL_OP_TASK(mask, op, task, args...) \
1149 do { \
1150 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \
1151 current->scx.kf_tasks[0] = task; \
1152 SCX_CALL_OP(mask, op, task, ##args); \
1153 current->scx.kf_tasks[0] = NULL; \
1154 } while (0)
1155
1156 #define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \
1157 ({ \
1158 __typeof__(scx_ops.op(task, ##args)) __ret; \
1159 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \
1160 current->scx.kf_tasks[0] = task; \
1161 __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \
1162 current->scx.kf_tasks[0] = NULL; \
1163 __ret; \
1164 })
1165
1166 #define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \
1167 ({ \
1168 __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \
1169 BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \
1170 current->scx.kf_tasks[0] = task0; \
1171 current->scx.kf_tasks[1] = task1; \
1172 __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \
1173 current->scx.kf_tasks[0] = NULL; \
1174 current->scx.kf_tasks[1] = NULL; \
1175 __ret; \
1176 })
1177
1178 /* @mask is constant, always inline to cull unnecessary branches */
scx_kf_allowed(u32 mask)1179 static __always_inline bool scx_kf_allowed(u32 mask)
1180 {
1181 if (unlikely(!(current->scx.kf_mask & mask))) {
1182 scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x",
1183 mask, current->scx.kf_mask);
1184 return false;
1185 }
1186
1187 /*
1188 * Enforce nesting boundaries. e.g. A kfunc which can be called from
1189 * DISPATCH must not be called if we're running DEQUEUE which is nested
1190 * inside ops.dispatch(). We don't need to check boundaries for any
1191 * blocking kfuncs as the verifier ensures they're only called from
1192 * sleepable progs.
1193 */
1194 if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE &&
1195 (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) {
1196 scx_ops_error("cpu_release kfunc called from a nested operation");
1197 return false;
1198 }
1199
1200 if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH &&
1201 (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) {
1202 scx_ops_error("dispatch kfunc called from a nested operation");
1203 return false;
1204 }
1205
1206 return true;
1207 }
1208
1209 /* see SCX_CALL_OP_TASK() */
scx_kf_allowed_on_arg_tasks(u32 mask,struct task_struct * p)1210 static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask,
1211 struct task_struct *p)
1212 {
1213 if (!scx_kf_allowed(mask))
1214 return false;
1215
1216 if (unlikely((p != current->scx.kf_tasks[0] &&
1217 p != current->scx.kf_tasks[1]))) {
1218 scx_ops_error("called on a task not being operated on");
1219 return false;
1220 }
1221
1222 return true;
1223 }
1224
scx_kf_allowed_if_unlocked(void)1225 static bool scx_kf_allowed_if_unlocked(void)
1226 {
1227 return !current->scx.kf_mask;
1228 }
1229
1230 /**
1231 * nldsq_next_task - Iterate to the next task in a non-local DSQ
1232 * @dsq: user dsq being iterated
1233 * @cur: current position, %NULL to start iteration
1234 * @rev: walk backwards
1235 *
1236 * Returns %NULL when iteration is finished.
1237 */
nldsq_next_task(struct scx_dispatch_q * dsq,struct task_struct * cur,bool rev)1238 static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq,
1239 struct task_struct *cur, bool rev)
1240 {
1241 struct list_head *list_node;
1242 struct scx_dsq_list_node *dsq_lnode;
1243
1244 lockdep_assert_held(&dsq->lock);
1245
1246 if (cur)
1247 list_node = &cur->scx.dsq_list.node;
1248 else
1249 list_node = &dsq->list;
1250
1251 /* find the next task, need to skip BPF iteration cursors */
1252 do {
1253 if (rev)
1254 list_node = list_node->prev;
1255 else
1256 list_node = list_node->next;
1257
1258 if (list_node == &dsq->list)
1259 return NULL;
1260
1261 dsq_lnode = container_of(list_node, struct scx_dsq_list_node,
1262 node);
1263 } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR);
1264
1265 return container_of(dsq_lnode, struct task_struct, scx.dsq_list);
1266 }
1267
1268 #define nldsq_for_each_task(p, dsq) \
1269 for ((p) = nldsq_next_task((dsq), NULL, false); (p); \
1270 (p) = nldsq_next_task((dsq), (p), false))
1271
1272
1273 /*
1274 * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse]
1275 * dispatch order. BPF-visible iterator is opaque and larger to allow future
1276 * changes without breaking backward compatibility. Can be used with
1277 * bpf_for_each(). See bpf_iter_scx_dsq_*().
1278 */
1279 enum scx_dsq_iter_flags {
1280 /* iterate in the reverse dispatch order */
1281 SCX_DSQ_ITER_REV = 1U << 16,
1282
1283 __SCX_DSQ_ITER_HAS_SLICE = 1U << 30,
1284 __SCX_DSQ_ITER_HAS_VTIME = 1U << 31,
1285
1286 __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV,
1287 __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS |
1288 __SCX_DSQ_ITER_HAS_SLICE |
1289 __SCX_DSQ_ITER_HAS_VTIME,
1290 };
1291
1292 struct bpf_iter_scx_dsq_kern {
1293 struct scx_dsq_list_node cursor;
1294 struct scx_dispatch_q *dsq;
1295 u64 slice;
1296 u64 vtime;
1297 } __attribute__((aligned(8)));
1298
1299 struct bpf_iter_scx_dsq {
1300 u64 __opaque[6];
1301 } __attribute__((aligned(8)));
1302
1303
1304 /*
1305 * SCX task iterator.
1306 */
1307 struct scx_task_iter {
1308 struct sched_ext_entity cursor;
1309 struct task_struct *locked;
1310 struct rq *rq;
1311 struct rq_flags rf;
1312 u32 cnt;
1313 };
1314
1315 /**
1316 * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration
1317 * @iter: iterator to init
1318 *
1319 * Initialize @iter and return with scx_tasks_lock held. Once initialized, @iter
1320 * must eventually be stopped with scx_task_iter_stop().
1321 *
1322 * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock()
1323 * between this and the first next() call or between any two next() calls. If
1324 * the locks are released between two next() calls, the caller is responsible
1325 * for ensuring that the task being iterated remains accessible either through
1326 * RCU read lock or obtaining a reference count.
1327 *
1328 * All tasks which existed when the iteration started are guaranteed to be
1329 * visited as long as they still exist.
1330 */
scx_task_iter_start(struct scx_task_iter * iter)1331 static void scx_task_iter_start(struct scx_task_iter *iter)
1332 {
1333 BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS &
1334 ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1));
1335
1336 spin_lock_irq(&scx_tasks_lock);
1337
1338 iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
1339 list_add(&iter->cursor.tasks_node, &scx_tasks);
1340 iter->locked = NULL;
1341 iter->cnt = 0;
1342 }
1343
__scx_task_iter_rq_unlock(struct scx_task_iter * iter)1344 static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter)
1345 {
1346 if (iter->locked) {
1347 task_rq_unlock(iter->rq, iter->locked, &iter->rf);
1348 iter->locked = NULL;
1349 }
1350 }
1351
1352 /**
1353 * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator
1354 * @iter: iterator to unlock
1355 *
1356 * If @iter is in the middle of a locked iteration, it may be locking the rq of
1357 * the task currently being visited in addition to scx_tasks_lock. Unlock both.
1358 * This function can be safely called anytime during an iteration.
1359 */
scx_task_iter_unlock(struct scx_task_iter * iter)1360 static void scx_task_iter_unlock(struct scx_task_iter *iter)
1361 {
1362 __scx_task_iter_rq_unlock(iter);
1363 spin_unlock_irq(&scx_tasks_lock);
1364 }
1365
1366 /**
1367 * scx_task_iter_relock - Lock scx_tasks_lock released by scx_task_iter_unlock()
1368 * @iter: iterator to re-lock
1369 *
1370 * Re-lock scx_tasks_lock unlocked by scx_task_iter_unlock(). Note that it
1371 * doesn't re-lock the rq lock. Must be called before other iterator operations.
1372 */
scx_task_iter_relock(struct scx_task_iter * iter)1373 static void scx_task_iter_relock(struct scx_task_iter *iter)
1374 {
1375 spin_lock_irq(&scx_tasks_lock);
1376 }
1377
1378 /**
1379 * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock
1380 * @iter: iterator to exit
1381 *
1382 * Exit a previously initialized @iter. Must be called with scx_tasks_lock held
1383 * which is released on return. If the iterator holds a task's rq lock, that rq
1384 * lock is also released. See scx_task_iter_start() for details.
1385 */
scx_task_iter_stop(struct scx_task_iter * iter)1386 static void scx_task_iter_stop(struct scx_task_iter *iter)
1387 {
1388 list_del_init(&iter->cursor.tasks_node);
1389 scx_task_iter_unlock(iter);
1390 }
1391
1392 /**
1393 * scx_task_iter_next - Next task
1394 * @iter: iterator to walk
1395 *
1396 * Visit the next task. See scx_task_iter_start() for details. Locks are dropped
1397 * and re-acquired every %SCX_OPS_TASK_ITER_BATCH iterations to avoid causing
1398 * stalls by holding scx_tasks_lock for too long.
1399 */
scx_task_iter_next(struct scx_task_iter * iter)1400 static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)
1401 {
1402 struct list_head *cursor = &iter->cursor.tasks_node;
1403 struct sched_ext_entity *pos;
1404
1405 if (!(++iter->cnt % SCX_OPS_TASK_ITER_BATCH)) {
1406 scx_task_iter_unlock(iter);
1407 cond_resched();
1408 scx_task_iter_relock(iter);
1409 }
1410
1411 list_for_each_entry(pos, cursor, tasks_node) {
1412 if (&pos->tasks_node == &scx_tasks)
1413 return NULL;
1414 if (!(pos->flags & SCX_TASK_CURSOR)) {
1415 list_move(cursor, &pos->tasks_node);
1416 return container_of(pos, struct task_struct, scx);
1417 }
1418 }
1419
1420 /* can't happen, should always terminate at scx_tasks above */
1421 BUG();
1422 }
1423
1424 /**
1425 * scx_task_iter_next_locked - Next non-idle task with its rq locked
1426 * @iter: iterator to walk
1427 *
1428 * Visit the non-idle task with its rq lock held. Allows callers to specify
1429 * whether they would like to filter out dead tasks. See scx_task_iter_start()
1430 * for details.
1431 */
scx_task_iter_next_locked(struct scx_task_iter * iter)1432 static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
1433 {
1434 struct task_struct *p;
1435
1436 __scx_task_iter_rq_unlock(iter);
1437
1438 while ((p = scx_task_iter_next(iter))) {
1439 /*
1440 * scx_task_iter is used to prepare and move tasks into SCX
1441 * while loading the BPF scheduler and vice-versa while
1442 * unloading. The init_tasks ("swappers") should be excluded
1443 * from the iteration because:
1444 *
1445 * - It's unsafe to use __setschduler_prio() on an init_task to
1446 * determine the sched_class to use as it won't preserve its
1447 * idle_sched_class.
1448 *
1449 * - ops.init/exit_task() can easily be confused if called with
1450 * init_tasks as they, e.g., share PID 0.
1451 *
1452 * As init_tasks are never scheduled through SCX, they can be
1453 * skipped safely. Note that is_idle_task() which tests %PF_IDLE
1454 * doesn't work here:
1455 *
1456 * - %PF_IDLE may not be set for an init_task whose CPU hasn't
1457 * yet been onlined.
1458 *
1459 * - %PF_IDLE can be set on tasks that are not init_tasks. See
1460 * play_idle_precise() used by CONFIG_IDLE_INJECT.
1461 *
1462 * Test for idle_sched_class as only init_tasks are on it.
1463 */
1464 if (p->sched_class != &idle_sched_class)
1465 break;
1466 }
1467 if (!p)
1468 return NULL;
1469
1470 iter->rq = task_rq_lock(p, &iter->rf);
1471 iter->locked = p;
1472
1473 return p;
1474 }
1475
scx_ops_enable_state(void)1476 static enum scx_ops_enable_state scx_ops_enable_state(void)
1477 {
1478 return atomic_read(&scx_ops_enable_state_var);
1479 }
1480
1481 static enum scx_ops_enable_state
scx_ops_set_enable_state(enum scx_ops_enable_state to)1482 scx_ops_set_enable_state(enum scx_ops_enable_state to)
1483 {
1484 return atomic_xchg(&scx_ops_enable_state_var, to);
1485 }
1486
scx_ops_tryset_enable_state(enum scx_ops_enable_state to,enum scx_ops_enable_state from)1487 static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to,
1488 enum scx_ops_enable_state from)
1489 {
1490 int from_v = from;
1491
1492 return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to);
1493 }
1494
scx_rq_bypassing(struct rq * rq)1495 static bool scx_rq_bypassing(struct rq *rq)
1496 {
1497 return unlikely(rq->scx.flags & SCX_RQ_BYPASSING);
1498 }
1499
1500 /**
1501 * wait_ops_state - Busy-wait the specified ops state to end
1502 * @p: target task
1503 * @opss: state to wait the end of
1504 *
1505 * Busy-wait for @p to transition out of @opss. This can only be used when the
1506 * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also
1507 * has load_acquire semantics to ensure that the caller can see the updates made
1508 * in the enqueueing and dispatching paths.
1509 */
wait_ops_state(struct task_struct * p,unsigned long opss)1510 static void wait_ops_state(struct task_struct *p, unsigned long opss)
1511 {
1512 do {
1513 cpu_relax();
1514 } while (atomic_long_read_acquire(&p->scx.ops_state) == opss);
1515 }
1516
1517 /**
1518 * ops_cpu_valid - Verify a cpu number
1519 * @cpu: cpu number which came from a BPF ops
1520 * @where: extra information reported on error
1521 *
1522 * @cpu is a cpu number which came from the BPF scheduler and can be any value.
1523 * Verify that it is in range and one of the possible cpus. If invalid, trigger
1524 * an ops error.
1525 */
ops_cpu_valid(s32 cpu,const char * where)1526 static bool ops_cpu_valid(s32 cpu, const char *where)
1527 {
1528 if (likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu))) {
1529 return true;
1530 } else {
1531 scx_ops_error("invalid CPU %d%s%s", cpu,
1532 where ? " " : "", where ?: "");
1533 return false;
1534 }
1535 }
1536
1537 /**
1538 * ops_sanitize_err - Sanitize a -errno value
1539 * @ops_name: operation to blame on failure
1540 * @err: -errno value to sanitize
1541 *
1542 * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return
1543 * -%EPROTO. This is necessary because returning a rogue -errno up the chain can
1544 * cause misbehaviors. For an example, a large negative return from
1545 * ops.init_task() triggers an oops when passed up the call chain because the
1546 * value fails IS_ERR() test after being encoded with ERR_PTR() and then is
1547 * handled as a pointer.
1548 */
ops_sanitize_err(const char * ops_name,s32 err)1549 static int ops_sanitize_err(const char *ops_name, s32 err)
1550 {
1551 if (err < 0 && err >= -MAX_ERRNO)
1552 return err;
1553
1554 scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err);
1555 return -EPROTO;
1556 }
1557
run_deferred(struct rq * rq)1558 static void run_deferred(struct rq *rq)
1559 {
1560 process_ddsp_deferred_locals(rq);
1561 }
1562
1563 #ifdef CONFIG_SMP
deferred_bal_cb_workfn(struct rq * rq)1564 static void deferred_bal_cb_workfn(struct rq *rq)
1565 {
1566 run_deferred(rq);
1567 }
1568 #endif
1569
deferred_irq_workfn(struct irq_work * irq_work)1570 static void deferred_irq_workfn(struct irq_work *irq_work)
1571 {
1572 struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work);
1573
1574 raw_spin_rq_lock(rq);
1575 run_deferred(rq);
1576 raw_spin_rq_unlock(rq);
1577 }
1578
1579 /**
1580 * schedule_deferred - Schedule execution of deferred actions on an rq
1581 * @rq: target rq
1582 *
1583 * Schedule execution of deferred actions on @rq. Must be called with @rq
1584 * locked. Deferred actions are executed with @rq locked but unpinned, and thus
1585 * can unlock @rq to e.g. migrate tasks to other rqs.
1586 */
schedule_deferred(struct rq * rq)1587 static void schedule_deferred(struct rq *rq)
1588 {
1589 lockdep_assert_rq_held(rq);
1590
1591 #ifdef CONFIG_SMP
1592 /*
1593 * If in the middle of waking up a task, task_woken_scx() will be called
1594 * afterwards which will then run the deferred actions, no need to
1595 * schedule anything.
1596 */
1597 if (rq->scx.flags & SCX_RQ_IN_WAKEUP)
1598 return;
1599
1600 /*
1601 * If in balance, the balance callbacks will be called before rq lock is
1602 * released. Schedule one.
1603 */
1604 if (rq->scx.flags & SCX_RQ_IN_BALANCE) {
1605 queue_balance_callback(rq, &rq->scx.deferred_bal_cb,
1606 deferred_bal_cb_workfn);
1607 return;
1608 }
1609 #endif
1610 /*
1611 * No scheduler hooks available. Queue an irq work. They are executed on
1612 * IRQ re-enable which may take a bit longer than the scheduler hooks.
1613 * The above WAKEUP and BALANCE paths should cover most of the cases and
1614 * the time to IRQ re-enable shouldn't be long.
1615 */
1616 irq_work_queue(&rq->scx.deferred_irq_work);
1617 }
1618
1619 /**
1620 * touch_core_sched - Update timestamp used for core-sched task ordering
1621 * @rq: rq to read clock from, must be locked
1622 * @p: task to update the timestamp for
1623 *
1624 * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to
1625 * implement global or local-DSQ FIFO ordering for core-sched. Should be called
1626 * when a task becomes runnable and its turn on the CPU ends (e.g. slice
1627 * exhaustion).
1628 */
touch_core_sched(struct rq * rq,struct task_struct * p)1629 static void touch_core_sched(struct rq *rq, struct task_struct *p)
1630 {
1631 lockdep_assert_rq_held(rq);
1632
1633 #ifdef CONFIG_SCHED_CORE
1634 /*
1635 * It's okay to update the timestamp spuriously. Use
1636 * sched_core_disabled() which is cheaper than enabled().
1637 *
1638 * As this is used to determine ordering between tasks of sibling CPUs,
1639 * it may be better to use per-core dispatch sequence instead.
1640 */
1641 if (!sched_core_disabled())
1642 p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq));
1643 #endif
1644 }
1645
1646 /**
1647 * touch_core_sched_dispatch - Update core-sched timestamp on dispatch
1648 * @rq: rq to read clock from, must be locked
1649 * @p: task being dispatched
1650 *
1651 * If the BPF scheduler implements custom core-sched ordering via
1652 * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO
1653 * ordering within each local DSQ. This function is called from dispatch paths
1654 * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect.
1655 */
touch_core_sched_dispatch(struct rq * rq,struct task_struct * p)1656 static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p)
1657 {
1658 lockdep_assert_rq_held(rq);
1659
1660 #ifdef CONFIG_SCHED_CORE
1661 if (SCX_HAS_OP(core_sched_before))
1662 touch_core_sched(rq, p);
1663 #endif
1664 }
1665
update_curr_scx(struct rq * rq)1666 static void update_curr_scx(struct rq *rq)
1667 {
1668 struct task_struct *curr = rq->curr;
1669 s64 delta_exec;
1670
1671 delta_exec = update_curr_common(rq);
1672 if (unlikely(delta_exec <= 0))
1673 return;
1674
1675 if (curr->scx.slice != SCX_SLICE_INF) {
1676 curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec);
1677 if (!curr->scx.slice)
1678 touch_core_sched(rq, curr);
1679 }
1680 }
1681
scx_dsq_priq_less(struct rb_node * node_a,const struct rb_node * node_b)1682 static bool scx_dsq_priq_less(struct rb_node *node_a,
1683 const struct rb_node *node_b)
1684 {
1685 const struct task_struct *a =
1686 container_of(node_a, struct task_struct, scx.dsq_priq);
1687 const struct task_struct *b =
1688 container_of(node_b, struct task_struct, scx.dsq_priq);
1689
1690 return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime);
1691 }
1692
dsq_mod_nr(struct scx_dispatch_q * dsq,s32 delta)1693 static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta)
1694 {
1695 /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */
1696 WRITE_ONCE(dsq->nr, dsq->nr + delta);
1697 }
1698
dispatch_enqueue(struct scx_dispatch_q * dsq,struct task_struct * p,u64 enq_flags)1699 static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p,
1700 u64 enq_flags)
1701 {
1702 bool is_local = dsq->id == SCX_DSQ_LOCAL;
1703
1704 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
1705 WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
1706 !RB_EMPTY_NODE(&p->scx.dsq_priq));
1707
1708 if (!is_local) {
1709 raw_spin_lock(&dsq->lock);
1710 if (unlikely(dsq->id == SCX_DSQ_INVALID)) {
1711 scx_ops_error("attempting to dispatch to a destroyed dsq");
1712 /* fall back to the global dsq */
1713 raw_spin_unlock(&dsq->lock);
1714 dsq = find_global_dsq(p);
1715 raw_spin_lock(&dsq->lock);
1716 }
1717 }
1718
1719 if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) &&
1720 (enq_flags & SCX_ENQ_DSQ_PRIQ))) {
1721 /*
1722 * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from
1723 * their FIFO queues. To avoid confusion and accidentally
1724 * starving vtime-dispatched tasks by FIFO-dispatched tasks, we
1725 * disallow any internal DSQ from doing vtime ordering of
1726 * tasks.
1727 */
1728 scx_ops_error("cannot use vtime ordering for built-in DSQs");
1729 enq_flags &= ~SCX_ENQ_DSQ_PRIQ;
1730 }
1731
1732 if (enq_flags & SCX_ENQ_DSQ_PRIQ) {
1733 struct rb_node *rbp;
1734
1735 /*
1736 * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are
1737 * linked to both the rbtree and list on PRIQs, this can only be
1738 * tested easily when adding the first task.
1739 */
1740 if (unlikely(RB_EMPTY_ROOT(&dsq->priq) &&
1741 nldsq_next_task(dsq, NULL, false)))
1742 scx_ops_error("DSQ ID 0x%016llx already had FIFO-enqueued tasks",
1743 dsq->id);
1744
1745 p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ;
1746 rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less);
1747
1748 /*
1749 * Find the previous task and insert after it on the list so
1750 * that @dsq->list is vtime ordered.
1751 */
1752 rbp = rb_prev(&p->scx.dsq_priq);
1753 if (rbp) {
1754 struct task_struct *prev =
1755 container_of(rbp, struct task_struct,
1756 scx.dsq_priq);
1757 list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node);
1758 } else {
1759 list_add(&p->scx.dsq_list.node, &dsq->list);
1760 }
1761 } else {
1762 /* a FIFO DSQ shouldn't be using PRIQ enqueuing */
1763 if (unlikely(!RB_EMPTY_ROOT(&dsq->priq)))
1764 scx_ops_error("DSQ ID 0x%016llx already had PRIQ-enqueued tasks",
1765 dsq->id);
1766
1767 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
1768 list_add(&p->scx.dsq_list.node, &dsq->list);
1769 else
1770 list_add_tail(&p->scx.dsq_list.node, &dsq->list);
1771 }
1772
1773 /* seq records the order tasks are queued, used by BPF DSQ iterator */
1774 dsq->seq++;
1775 p->scx.dsq_seq = dsq->seq;
1776
1777 dsq_mod_nr(dsq, 1);
1778 p->scx.dsq = dsq;
1779
1780 /*
1781 * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the
1782 * direct dispatch path, but we clear them here because the direct
1783 * dispatch verdict may be overridden on the enqueue path during e.g.
1784 * bypass.
1785 */
1786 p->scx.ddsp_dsq_id = SCX_DSQ_INVALID;
1787 p->scx.ddsp_enq_flags = 0;
1788
1789 /*
1790 * We're transitioning out of QUEUEING or DISPATCHING. store_release to
1791 * match waiters' load_acquire.
1792 */
1793 if (enq_flags & SCX_ENQ_CLEAR_OPSS)
1794 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
1795
1796 if (is_local) {
1797 struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1798 bool preempt = false;
1799
1800 if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
1801 rq->curr->sched_class == &ext_sched_class) {
1802 rq->curr->scx.slice = 0;
1803 preempt = true;
1804 }
1805
1806 if (preempt || sched_class_above(&ext_sched_class,
1807 rq->curr->sched_class))
1808 resched_curr(rq);
1809 } else {
1810 raw_spin_unlock(&dsq->lock);
1811 }
1812 }
1813
task_unlink_from_dsq(struct task_struct * p,struct scx_dispatch_q * dsq)1814 static void task_unlink_from_dsq(struct task_struct *p,
1815 struct scx_dispatch_q *dsq)
1816 {
1817 WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
1818
1819 if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) {
1820 rb_erase(&p->scx.dsq_priq, &dsq->priq);
1821 RB_CLEAR_NODE(&p->scx.dsq_priq);
1822 p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ;
1823 }
1824
1825 list_del_init(&p->scx.dsq_list.node);
1826 dsq_mod_nr(dsq, -1);
1827 }
1828
dispatch_dequeue(struct rq * rq,struct task_struct * p)1829 static void dispatch_dequeue(struct rq *rq, struct task_struct *p)
1830 {
1831 struct scx_dispatch_q *dsq = p->scx.dsq;
1832 bool is_local = dsq == &rq->scx.local_dsq;
1833
1834 if (!dsq) {
1835 /*
1836 * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals.
1837 * Unlinking is all that's needed to cancel.
1838 */
1839 if (unlikely(!list_empty(&p->scx.dsq_list.node)))
1840 list_del_init(&p->scx.dsq_list.node);
1841
1842 /*
1843 * When dispatching directly from the BPF scheduler to a local
1844 * DSQ, the task isn't associated with any DSQ but
1845 * @p->scx.holding_cpu may be set under the protection of
1846 * %SCX_OPSS_DISPATCHING.
1847 */
1848 if (p->scx.holding_cpu >= 0)
1849 p->scx.holding_cpu = -1;
1850
1851 return;
1852 }
1853
1854 if (!is_local)
1855 raw_spin_lock(&dsq->lock);
1856
1857 /*
1858 * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't
1859 * change underneath us.
1860 */
1861 if (p->scx.holding_cpu < 0) {
1862 /* @p must still be on @dsq, dequeue */
1863 task_unlink_from_dsq(p, dsq);
1864 } else {
1865 /*
1866 * We're racing against dispatch_to_local_dsq() which already
1867 * removed @p from @dsq and set @p->scx.holding_cpu. Clear the
1868 * holding_cpu which tells dispatch_to_local_dsq() that it lost
1869 * the race.
1870 */
1871 WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
1872 p->scx.holding_cpu = -1;
1873 }
1874 p->scx.dsq = NULL;
1875
1876 if (!is_local)
1877 raw_spin_unlock(&dsq->lock);
1878 }
1879
find_dsq_for_dispatch(struct rq * rq,u64 dsq_id,struct task_struct * p)1880 static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id,
1881 struct task_struct *p)
1882 {
1883 struct scx_dispatch_q *dsq;
1884
1885 if (dsq_id == SCX_DSQ_LOCAL)
1886 return &rq->scx.local_dsq;
1887
1888 if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
1889 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
1890
1891 if (!ops_cpu_valid(cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict"))
1892 return find_global_dsq(p);
1893
1894 return &cpu_rq(cpu)->scx.local_dsq;
1895 }
1896
1897 if (dsq_id == SCX_DSQ_GLOBAL)
1898 dsq = find_global_dsq(p);
1899 else
1900 dsq = find_user_dsq(dsq_id);
1901
1902 if (unlikely(!dsq)) {
1903 scx_ops_error("non-existent DSQ 0x%llx for %s[%d]",
1904 dsq_id, p->comm, p->pid);
1905 return find_global_dsq(p);
1906 }
1907
1908 return dsq;
1909 }
1910
mark_direct_dispatch(struct task_struct * ddsp_task,struct task_struct * p,u64 dsq_id,u64 enq_flags)1911 static void mark_direct_dispatch(struct task_struct *ddsp_task,
1912 struct task_struct *p, u64 dsq_id,
1913 u64 enq_flags)
1914 {
1915 /*
1916 * Mark that dispatch already happened from ops.select_cpu() or
1917 * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value
1918 * which can never match a valid task pointer.
1919 */
1920 __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH));
1921
1922 /* @p must match the task on the enqueue path */
1923 if (unlikely(p != ddsp_task)) {
1924 if (IS_ERR(ddsp_task))
1925 scx_ops_error("%s[%d] already direct-dispatched",
1926 p->comm, p->pid);
1927 else
1928 scx_ops_error("scheduling for %s[%d] but trying to direct-dispatch %s[%d]",
1929 ddsp_task->comm, ddsp_task->pid,
1930 p->comm, p->pid);
1931 return;
1932 }
1933
1934 WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID);
1935 WARN_ON_ONCE(p->scx.ddsp_enq_flags);
1936
1937 p->scx.ddsp_dsq_id = dsq_id;
1938 p->scx.ddsp_enq_flags = enq_flags;
1939 }
1940
direct_dispatch(struct task_struct * p,u64 enq_flags)1941 static void direct_dispatch(struct task_struct *p, u64 enq_flags)
1942 {
1943 struct rq *rq = task_rq(p);
1944 struct scx_dispatch_q *dsq =
1945 find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p);
1946
1947 touch_core_sched_dispatch(rq, p);
1948
1949 p->scx.ddsp_enq_flags |= enq_flags;
1950
1951 /*
1952 * We are in the enqueue path with @rq locked and pinned, and thus can't
1953 * double lock a remote rq and enqueue to its local DSQ. For
1954 * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer
1955 * the enqueue so that it's executed when @rq can be unlocked.
1956 */
1957 if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) {
1958 unsigned long opss;
1959
1960 opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK;
1961
1962 switch (opss & SCX_OPSS_STATE_MASK) {
1963 case SCX_OPSS_NONE:
1964 break;
1965 case SCX_OPSS_QUEUEING:
1966 /*
1967 * As @p was never passed to the BPF side, _release is
1968 * not strictly necessary. Still do it for consistency.
1969 */
1970 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
1971 break;
1972 default:
1973 WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()",
1974 p->comm, p->pid, opss);
1975 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
1976 break;
1977 }
1978
1979 WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
1980 list_add_tail(&p->scx.dsq_list.node,
1981 &rq->scx.ddsp_deferred_locals);
1982 schedule_deferred(rq);
1983 return;
1984 }
1985
1986 dispatch_enqueue(dsq, p, p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS);
1987 }
1988
scx_rq_online(struct rq * rq)1989 static bool scx_rq_online(struct rq *rq)
1990 {
1991 /*
1992 * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates
1993 * the online state as seen from the BPF scheduler. cpu_active() test
1994 * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will
1995 * stay set until the current scheduling operation is complete even if
1996 * we aren't locking @rq.
1997 */
1998 return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));
1999 }
2000
do_enqueue_task(struct rq * rq,struct task_struct * p,u64 enq_flags,int sticky_cpu)2001 static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
2002 int sticky_cpu)
2003 {
2004 struct task_struct **ddsp_taskp;
2005 unsigned long qseq;
2006
2007 WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
2008
2009 /* rq migration */
2010 if (sticky_cpu == cpu_of(rq))
2011 goto local_norefill;
2012
2013 /*
2014 * If !scx_rq_online(), we already told the BPF scheduler that the CPU
2015 * is offline and are just running the hotplug path. Don't bother the
2016 * BPF scheduler.
2017 */
2018 if (!scx_rq_online(rq))
2019 goto local;
2020
2021 if (scx_rq_bypassing(rq))
2022 goto global;
2023
2024 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
2025 goto direct;
2026
2027 /* see %SCX_OPS_ENQ_EXITING */
2028 if (!static_branch_unlikely(&scx_ops_enq_exiting) &&
2029 unlikely(p->flags & PF_EXITING))
2030 goto local;
2031
2032 /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
2033 if (!static_branch_unlikely(&scx_ops_enq_migration_disabled) &&
2034 is_migration_disabled(p))
2035 goto local;
2036
2037 if (!SCX_HAS_OP(enqueue))
2038 goto global;
2039
2040 /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */
2041 qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
2042
2043 WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
2044 atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq);
2045
2046 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
2047 WARN_ON_ONCE(*ddsp_taskp);
2048 *ddsp_taskp = p;
2049
2050 SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags);
2051
2052 *ddsp_taskp = NULL;
2053 if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
2054 goto direct;
2055
2056 /*
2057 * If not directly dispatched, QUEUEING isn't clear yet and dispatch or
2058 * dequeue may be waiting. The store_release matches their load_acquire.
2059 */
2060 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq);
2061 return;
2062
2063 direct:
2064 direct_dispatch(p, enq_flags);
2065 return;
2066
2067 local:
2068 /*
2069 * For task-ordering, slice refill must be treated as implying the end
2070 * of the current slice. Otherwise, the longer @p stays on the CPU, the
2071 * higher priority it becomes from scx_prio_less()'s POV.
2072 */
2073 touch_core_sched(rq, p);
2074 p->scx.slice = SCX_SLICE_DFL;
2075 local_norefill:
2076 dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags);
2077 return;
2078
2079 global:
2080 touch_core_sched(rq, p); /* see the comment in local: */
2081 p->scx.slice = SCX_SLICE_DFL;
2082 dispatch_enqueue(find_global_dsq(p), p, enq_flags);
2083 }
2084
task_runnable(const struct task_struct * p)2085 static bool task_runnable(const struct task_struct *p)
2086 {
2087 return !list_empty(&p->scx.runnable_node);
2088 }
2089
set_task_runnable(struct rq * rq,struct task_struct * p)2090 static void set_task_runnable(struct rq *rq, struct task_struct *p)
2091 {
2092 lockdep_assert_rq_held(rq);
2093
2094 if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) {
2095 p->scx.runnable_at = jiffies;
2096 p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT;
2097 }
2098
2099 /*
2100 * list_add_tail() must be used. scx_ops_bypass() depends on tasks being
2101 * appended to the runnable_list.
2102 */
2103 list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
2104 }
2105
clr_task_runnable(struct task_struct * p,bool reset_runnable_at)2106 static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at)
2107 {
2108 list_del_init(&p->scx.runnable_node);
2109 if (reset_runnable_at)
2110 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
2111 }
2112
enqueue_task_scx(struct rq * rq,struct task_struct * p,int enq_flags)2113 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags)
2114 {
2115 int sticky_cpu = p->scx.sticky_cpu;
2116
2117 if (enq_flags & ENQUEUE_WAKEUP)
2118 rq->scx.flags |= SCX_RQ_IN_WAKEUP;
2119
2120 enq_flags |= rq->scx.extra_enq_flags;
2121
2122 if (sticky_cpu >= 0)
2123 p->scx.sticky_cpu = -1;
2124
2125 /*
2126 * Restoring a running task will be immediately followed by
2127 * set_next_task_scx() which expects the task to not be on the BPF
2128 * scheduler as tasks can only start running through local DSQs. Force
2129 * direct-dispatch into the local DSQ by setting the sticky_cpu.
2130 */
2131 if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p))
2132 sticky_cpu = cpu_of(rq);
2133
2134 if (p->scx.flags & SCX_TASK_QUEUED) {
2135 WARN_ON_ONCE(!task_runnable(p));
2136 goto out;
2137 }
2138
2139 set_task_runnable(rq, p);
2140 p->scx.flags |= SCX_TASK_QUEUED;
2141 rq->scx.nr_running++;
2142 add_nr_running(rq, 1);
2143
2144 if (SCX_HAS_OP(runnable) && !task_on_rq_migrating(p))
2145 SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags);
2146
2147 if (enq_flags & SCX_ENQ_WAKEUP)
2148 touch_core_sched(rq, p);
2149
2150 do_enqueue_task(rq, p, enq_flags, sticky_cpu);
2151 out:
2152 rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
2153 }
2154
ops_dequeue(struct task_struct * p,u64 deq_flags)2155 static void ops_dequeue(struct task_struct *p, u64 deq_flags)
2156 {
2157 unsigned long opss;
2158
2159 /* dequeue is always temporary, don't reset runnable_at */
2160 clr_task_runnable(p, false);
2161
2162 /* acquire ensures that we see the preceding updates on QUEUED */
2163 opss = atomic_long_read_acquire(&p->scx.ops_state);
2164
2165 switch (opss & SCX_OPSS_STATE_MASK) {
2166 case SCX_OPSS_NONE:
2167 break;
2168 case SCX_OPSS_QUEUEING:
2169 /*
2170 * QUEUEING is started and finished while holding @p's rq lock.
2171 * As we're holding the rq lock now, we shouldn't see QUEUEING.
2172 */
2173 BUG();
2174 case SCX_OPSS_QUEUED:
2175 if (SCX_HAS_OP(dequeue))
2176 SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags);
2177
2178 if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
2179 SCX_OPSS_NONE))
2180 break;
2181 fallthrough;
2182 case SCX_OPSS_DISPATCHING:
2183 /*
2184 * If @p is being dispatched from the BPF scheduler to a DSQ,
2185 * wait for the transfer to complete so that @p doesn't get
2186 * added to its DSQ after dequeueing is complete.
2187 *
2188 * As we're waiting on DISPATCHING with the rq locked, the
2189 * dispatching side shouldn't try to lock the rq while
2190 * DISPATCHING is set. See dispatch_to_local_dsq().
2191 *
2192 * DISPATCHING shouldn't have qseq set and control can reach
2193 * here with NONE @opss from the above QUEUED case block.
2194 * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss.
2195 */
2196 wait_ops_state(p, SCX_OPSS_DISPATCHING);
2197 BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
2198 break;
2199 }
2200 }
2201
dequeue_task_scx(struct rq * rq,struct task_struct * p,int deq_flags)2202 static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags)
2203 {
2204 if (!(p->scx.flags & SCX_TASK_QUEUED)) {
2205 WARN_ON_ONCE(task_runnable(p));
2206 return true;
2207 }
2208
2209 ops_dequeue(p, deq_flags);
2210
2211 /*
2212 * A currently running task which is going off @rq first gets dequeued
2213 * and then stops running. As we want running <-> stopping transitions
2214 * to be contained within runnable <-> quiescent transitions, trigger
2215 * ->stopping() early here instead of in put_prev_task_scx().
2216 *
2217 * @p may go through multiple stopping <-> running transitions between
2218 * here and put_prev_task_scx() if task attribute changes occur while
2219 * balance_scx() leaves @rq unlocked. However, they don't contain any
2220 * information meaningful to the BPF scheduler and can be suppressed by
2221 * skipping the callbacks if the task is !QUEUED.
2222 */
2223 if (SCX_HAS_OP(stopping) && task_current(rq, p)) {
2224 update_curr_scx(rq);
2225 SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false);
2226 }
2227
2228 if (SCX_HAS_OP(quiescent) && !task_on_rq_migrating(p))
2229 SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags);
2230
2231 if (deq_flags & SCX_DEQ_SLEEP)
2232 p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP;
2233 else
2234 p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP;
2235
2236 p->scx.flags &= ~SCX_TASK_QUEUED;
2237 rq->scx.nr_running--;
2238 sub_nr_running(rq, 1);
2239
2240 dispatch_dequeue(rq, p);
2241 return true;
2242 }
2243
yield_task_scx(struct rq * rq)2244 static void yield_task_scx(struct rq *rq)
2245 {
2246 struct task_struct *p = rq->curr;
2247
2248 if (SCX_HAS_OP(yield))
2249 SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL);
2250 else
2251 p->scx.slice = 0;
2252 }
2253
yield_to_task_scx(struct rq * rq,struct task_struct * to)2254 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
2255 {
2256 struct task_struct *from = rq->curr;
2257
2258 if (SCX_HAS_OP(yield))
2259 return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to);
2260 else
2261 return false;
2262 }
2263
move_local_task_to_local_dsq(struct task_struct * p,u64 enq_flags,struct scx_dispatch_q * src_dsq,struct rq * dst_rq)2264 static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
2265 struct scx_dispatch_q *src_dsq,
2266 struct rq *dst_rq)
2267 {
2268 struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq;
2269
2270 /* @dsq is locked and @p is on @dst_rq */
2271 lockdep_assert_held(&src_dsq->lock);
2272 lockdep_assert_rq_held(dst_rq);
2273
2274 WARN_ON_ONCE(p->scx.holding_cpu >= 0);
2275
2276 if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
2277 list_add(&p->scx.dsq_list.node, &dst_dsq->list);
2278 else
2279 list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
2280
2281 dsq_mod_nr(dst_dsq, 1);
2282 p->scx.dsq = dst_dsq;
2283 }
2284
2285 #ifdef CONFIG_SMP
2286 /**
2287 * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ
2288 * @p: task to move
2289 * @enq_flags: %SCX_ENQ_*
2290 * @src_rq: rq to move the task from, locked on entry, released on return
2291 * @dst_rq: rq to move the task into, locked on return
2292 *
2293 * Move @p which is currently on @src_rq to @dst_rq's local DSQ.
2294 */
move_remote_task_to_local_dsq(struct task_struct * p,u64 enq_flags,struct rq * src_rq,struct rq * dst_rq)2295 static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
2296 struct rq *src_rq, struct rq *dst_rq)
2297 {
2298 lockdep_assert_rq_held(src_rq);
2299
2300 /* the following marks @p MIGRATING which excludes dequeue */
2301 deactivate_task(src_rq, p, 0);
2302 set_task_cpu(p, cpu_of(dst_rq));
2303 p->scx.sticky_cpu = cpu_of(dst_rq);
2304
2305 raw_spin_rq_unlock(src_rq);
2306 raw_spin_rq_lock(dst_rq);
2307
2308 /*
2309 * We want to pass scx-specific enq_flags but activate_task() will
2310 * truncate the upper 32 bit. As we own @rq, we can pass them through
2311 * @rq->scx.extra_enq_flags instead.
2312 */
2313 WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr));
2314 WARN_ON_ONCE(dst_rq->scx.extra_enq_flags);
2315 dst_rq->scx.extra_enq_flags = enq_flags;
2316 activate_task(dst_rq, p, 0);
2317 dst_rq->scx.extra_enq_flags = 0;
2318 }
2319
2320 /*
2321 * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two
2322 * differences:
2323 *
2324 * - is_cpu_allowed() asks "Can this task run on this CPU?" while
2325 * task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to
2326 * this CPU?".
2327 *
2328 * While migration is disabled, is_cpu_allowed() has to say "yes" as the task
2329 * must be allowed to finish on the CPU that it's currently on regardless of
2330 * the CPU state. However, task_can_run_on_remote_rq() must say "no" as the
2331 * BPF scheduler shouldn't attempt to migrate a task which has migration
2332 * disabled.
2333 *
2334 * - The BPF scheduler is bypassed while the rq is offline and we can always say
2335 * no to the BPF scheduler initiated migrations while offline.
2336 *
2337 * The caller must ensure that @p and @rq are on different CPUs.
2338 */
task_can_run_on_remote_rq(struct task_struct * p,struct rq * rq,bool trigger_error)2339 static bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq,
2340 bool trigger_error)
2341 {
2342 int cpu = cpu_of(rq);
2343
2344 SCHED_WARN_ON(task_cpu(p) == cpu);
2345
2346 /*
2347 * If @p has migration disabled, @p->cpus_ptr is updated to contain only
2348 * the pinned CPU in migrate_disable_switch() while @p is being switched
2349 * out. However, put_prev_task_scx() is called before @p->cpus_ptr is
2350 * updated and thus another CPU may see @p on a DSQ inbetween leading to
2351 * @p passing the below task_allowed_on_cpu() check while migration is
2352 * disabled.
2353 *
2354 * Test the migration disabled state first as the race window is narrow
2355 * and the BPF scheduler failing to check migration disabled state can
2356 * easily be masked if task_allowed_on_cpu() is done first.
2357 */
2358 if (unlikely(is_migration_disabled(p))) {
2359 if (trigger_error)
2360 scx_ops_error("SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d",
2361 p->comm, p->pid, task_cpu(p), cpu);
2362 return false;
2363 }
2364
2365 /*
2366 * We don't require the BPF scheduler to avoid dispatching to offline
2367 * CPUs mostly for convenience but also because CPUs can go offline
2368 * between scx_bpf_dsq_insert() calls and here. Trigger error iff the
2369 * picked CPU is outside the allowed mask.
2370 */
2371 if (!task_allowed_on_cpu(p, cpu)) {
2372 if (trigger_error)
2373 scx_ops_error("SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]",
2374 cpu, p->comm, p->pid);
2375 return false;
2376 }
2377
2378 if (!scx_rq_online(rq))
2379 return false;
2380
2381 return true;
2382 }
2383
2384 /**
2385 * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq
2386 * @p: target task
2387 * @dsq: locked DSQ @p is currently on
2388 * @src_rq: rq @p is currently on, stable with @dsq locked
2389 *
2390 * Called with @dsq locked but no rq's locked. We want to move @p to a different
2391 * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is
2392 * required when transferring into a local DSQ. Even when transferring into a
2393 * non-local DSQ, it's better to use the same mechanism to protect against
2394 * dequeues and maintain the invariant that @p->scx.dsq can only change while
2395 * @src_rq is locked, which e.g. scx_dump_task() depends on.
2396 *
2397 * We want to grab @src_rq but that can deadlock if we try while locking @dsq,
2398 * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As
2399 * this may race with dequeue, which can't drop the rq lock or fail, do a little
2400 * dancing from our side.
2401 *
2402 * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets
2403 * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu
2404 * would be cleared to -1. While other cpus may have updated it to different
2405 * values afterwards, as this operation can't be preempted or recurse, the
2406 * holding_cpu can never become this CPU again before we're done. Thus, we can
2407 * tell whether we lost to dequeue by testing whether the holding_cpu still
2408 * points to this CPU. See dispatch_dequeue() for the counterpart.
2409 *
2410 * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is
2411 * still valid. %false if lost to dequeue.
2412 */
unlink_dsq_and_lock_src_rq(struct task_struct * p,struct scx_dispatch_q * dsq,struct rq * src_rq)2413 static bool unlink_dsq_and_lock_src_rq(struct task_struct *p,
2414 struct scx_dispatch_q *dsq,
2415 struct rq *src_rq)
2416 {
2417 s32 cpu = raw_smp_processor_id();
2418
2419 lockdep_assert_held(&dsq->lock);
2420
2421 WARN_ON_ONCE(p->scx.holding_cpu >= 0);
2422 task_unlink_from_dsq(p, dsq);
2423 p->scx.holding_cpu = cpu;
2424
2425 raw_spin_unlock(&dsq->lock);
2426 raw_spin_rq_lock(src_rq);
2427
2428 /* task_rq couldn't have changed if we're still the holding cpu */
2429 return likely(p->scx.holding_cpu == cpu) &&
2430 !WARN_ON_ONCE(src_rq != task_rq(p));
2431 }
2432
consume_remote_task(struct rq * this_rq,struct task_struct * p,struct scx_dispatch_q * dsq,struct rq * src_rq)2433 static bool consume_remote_task(struct rq *this_rq, struct task_struct *p,
2434 struct scx_dispatch_q *dsq, struct rq *src_rq)
2435 {
2436 raw_spin_rq_unlock(this_rq);
2437
2438 if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) {
2439 move_remote_task_to_local_dsq(p, 0, src_rq, this_rq);
2440 return true;
2441 } else {
2442 raw_spin_rq_unlock(src_rq);
2443 raw_spin_rq_lock(this_rq);
2444 return false;
2445 }
2446 }
2447 #else /* CONFIG_SMP */
move_remote_task_to_local_dsq(struct task_struct * p,u64 enq_flags,struct rq * src_rq,struct rq * dst_rq)2448 static inline void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { WARN_ON_ONCE(1); }
task_can_run_on_remote_rq(struct task_struct * p,struct rq * rq,bool trigger_error)2449 static inline bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq, bool trigger_error) { return false; }
consume_remote_task(struct rq * this_rq,struct task_struct * p,struct scx_dispatch_q * dsq,struct rq * task_rq)2450 static inline bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq) { return false; }
2451 #endif /* CONFIG_SMP */
2452
2453 /**
2454 * move_task_between_dsqs() - Move a task from one DSQ to another
2455 * @p: target task
2456 * @enq_flags: %SCX_ENQ_*
2457 * @src_dsq: DSQ @p is currently on, must not be a local DSQ
2458 * @dst_dsq: DSQ @p is being moved to, can be any DSQ
2459 *
2460 * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local
2461 * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq
2462 * will change. As @p's task_rq is locked, this function doesn't need to use the
2463 * holding_cpu mechanism.
2464 *
2465 * On return, @src_dsq is unlocked and only @p's new task_rq, which is the
2466 * return value, is locked.
2467 */
move_task_between_dsqs(struct task_struct * p,u64 enq_flags,struct scx_dispatch_q * src_dsq,struct scx_dispatch_q * dst_dsq)2468 static struct rq *move_task_between_dsqs(struct task_struct *p, u64 enq_flags,
2469 struct scx_dispatch_q *src_dsq,
2470 struct scx_dispatch_q *dst_dsq)
2471 {
2472 struct rq *src_rq = task_rq(p), *dst_rq;
2473
2474 BUG_ON(src_dsq->id == SCX_DSQ_LOCAL);
2475 lockdep_assert_held(&src_dsq->lock);
2476 lockdep_assert_rq_held(src_rq);
2477
2478 if (dst_dsq->id == SCX_DSQ_LOCAL) {
2479 dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
2480 if (src_rq != dst_rq &&
2481 unlikely(!task_can_run_on_remote_rq(p, dst_rq, true))) {
2482 dst_dsq = find_global_dsq(p);
2483 dst_rq = src_rq;
2484 }
2485 } else {
2486 /* no need to migrate if destination is a non-local DSQ */
2487 dst_rq = src_rq;
2488 }
2489
2490 /*
2491 * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different
2492 * CPU, @p will be migrated.
2493 */
2494 if (dst_dsq->id == SCX_DSQ_LOCAL) {
2495 /* @p is going from a non-local DSQ to a local DSQ */
2496 if (src_rq == dst_rq) {
2497 task_unlink_from_dsq(p, src_dsq);
2498 move_local_task_to_local_dsq(p, enq_flags,
2499 src_dsq, dst_rq);
2500 raw_spin_unlock(&src_dsq->lock);
2501 } else {
2502 raw_spin_unlock(&src_dsq->lock);
2503 move_remote_task_to_local_dsq(p, enq_flags,
2504 src_rq, dst_rq);
2505 }
2506 } else {
2507 /*
2508 * @p is going from a non-local DSQ to a non-local DSQ. As
2509 * $src_dsq is already locked, do an abbreviated dequeue.
2510 */
2511 task_unlink_from_dsq(p, src_dsq);
2512 p->scx.dsq = NULL;
2513 raw_spin_unlock(&src_dsq->lock);
2514
2515 dispatch_enqueue(dst_dsq, p, enq_flags);
2516 }
2517
2518 return dst_rq;
2519 }
2520
2521 /*
2522 * A poorly behaving BPF scheduler can live-lock the system by e.g. incessantly
2523 * banging on the same DSQ on a large NUMA system to the point where switching
2524 * to the bypass mode can take a long time. Inject artificial delays while the
2525 * bypass mode is switching to guarantee timely completion.
2526 */
scx_ops_breather(struct rq * rq)2527 static void scx_ops_breather(struct rq *rq)
2528 {
2529 u64 until;
2530
2531 lockdep_assert_rq_held(rq);
2532
2533 if (likely(!atomic_read(&scx_ops_breather_depth)))
2534 return;
2535
2536 raw_spin_rq_unlock(rq);
2537
2538 until = ktime_get_ns() + NSEC_PER_MSEC;
2539
2540 do {
2541 int cnt = 1024;
2542 while (atomic_read(&scx_ops_breather_depth) && --cnt)
2543 cpu_relax();
2544 } while (atomic_read(&scx_ops_breather_depth) &&
2545 time_before64(ktime_get_ns(), until));
2546
2547 raw_spin_rq_lock(rq);
2548 }
2549
consume_dispatch_q(struct rq * rq,struct scx_dispatch_q * dsq)2550 static bool consume_dispatch_q(struct rq *rq, struct scx_dispatch_q *dsq)
2551 {
2552 struct task_struct *p;
2553 retry:
2554 /*
2555 * This retry loop can repeatedly race against scx_ops_bypass()
2556 * dequeueing tasks from @dsq trying to put the system into the bypass
2557 * mode. On some multi-socket machines (e.g. 2x Intel 8480c), this can
2558 * live-lock the machine into soft lockups. Give a breather.
2559 */
2560 scx_ops_breather(rq);
2561
2562 /*
2563 * The caller can't expect to successfully consume a task if the task's
2564 * addition to @dsq isn't guaranteed to be visible somehow. Test
2565 * @dsq->list without locking and skip if it seems empty.
2566 */
2567 if (list_empty(&dsq->list))
2568 return false;
2569
2570 raw_spin_lock(&dsq->lock);
2571
2572 nldsq_for_each_task(p, dsq) {
2573 struct rq *task_rq = task_rq(p);
2574
2575 if (rq == task_rq) {
2576 task_unlink_from_dsq(p, dsq);
2577 move_local_task_to_local_dsq(p, 0, dsq, rq);
2578 raw_spin_unlock(&dsq->lock);
2579 return true;
2580 }
2581
2582 if (task_can_run_on_remote_rq(p, rq, false)) {
2583 if (likely(consume_remote_task(rq, p, dsq, task_rq)))
2584 return true;
2585 goto retry;
2586 }
2587 }
2588
2589 raw_spin_unlock(&dsq->lock);
2590 return false;
2591 }
2592
consume_global_dsq(struct rq * rq)2593 static bool consume_global_dsq(struct rq *rq)
2594 {
2595 int node = cpu_to_node(cpu_of(rq));
2596
2597 return consume_dispatch_q(rq, global_dsqs[node]);
2598 }
2599
2600 /**
2601 * dispatch_to_local_dsq - Dispatch a task to a local dsq
2602 * @rq: current rq which is locked
2603 * @dst_dsq: destination DSQ
2604 * @p: task to dispatch
2605 * @enq_flags: %SCX_ENQ_*
2606 *
2607 * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local
2608 * DSQ. This function performs all the synchronization dancing needed because
2609 * local DSQs are protected with rq locks.
2610 *
2611 * The caller must have exclusive ownership of @p (e.g. through
2612 * %SCX_OPSS_DISPATCHING).
2613 */
dispatch_to_local_dsq(struct rq * rq,struct scx_dispatch_q * dst_dsq,struct task_struct * p,u64 enq_flags)2614 static void dispatch_to_local_dsq(struct rq *rq, struct scx_dispatch_q *dst_dsq,
2615 struct task_struct *p, u64 enq_flags)
2616 {
2617 struct rq *src_rq = task_rq(p);
2618 struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
2619 #ifdef CONFIG_SMP
2620 struct rq *locked_rq = rq;
2621 #endif
2622
2623 /*
2624 * We're synchronized against dequeue through DISPATCHING. As @p can't
2625 * be dequeued, its task_rq and cpus_allowed are stable too.
2626 *
2627 * If dispatching to @rq that @p is already on, no lock dancing needed.
2628 */
2629 if (rq == src_rq && rq == dst_rq) {
2630 dispatch_enqueue(dst_dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
2631 return;
2632 }
2633
2634 #ifdef CONFIG_SMP
2635 if (src_rq != dst_rq &&
2636 unlikely(!task_can_run_on_remote_rq(p, dst_rq, true))) {
2637 dispatch_enqueue(find_global_dsq(p), p,
2638 enq_flags | SCX_ENQ_CLEAR_OPSS);
2639 return;
2640 }
2641
2642 /*
2643 * @p is on a possibly remote @src_rq which we need to lock to move the
2644 * task. If dequeue is in progress, it'd be locking @src_rq and waiting
2645 * on DISPATCHING, so we can't grab @src_rq lock while holding
2646 * DISPATCHING.
2647 *
2648 * As DISPATCHING guarantees that @p is wholly ours, we can pretend that
2649 * we're moving from a DSQ and use the same mechanism - mark the task
2650 * under transfer with holding_cpu, release DISPATCHING and then follow
2651 * the same protocol. See unlink_dsq_and_lock_src_rq().
2652 */
2653 p->scx.holding_cpu = raw_smp_processor_id();
2654
2655 /* store_release ensures that dequeue sees the above */
2656 atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
2657
2658 /* switch to @src_rq lock */
2659 if (locked_rq != src_rq) {
2660 raw_spin_rq_unlock(locked_rq);
2661 locked_rq = src_rq;
2662 raw_spin_rq_lock(src_rq);
2663 }
2664
2665 /* task_rq couldn't have changed if we're still the holding cpu */
2666 if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
2667 !WARN_ON_ONCE(src_rq != task_rq(p))) {
2668 /*
2669 * If @p is staying on the same rq, there's no need to go
2670 * through the full deactivate/activate cycle. Optimize by
2671 * abbreviating move_remote_task_to_local_dsq().
2672 */
2673 if (src_rq == dst_rq) {
2674 p->scx.holding_cpu = -1;
2675 dispatch_enqueue(&dst_rq->scx.local_dsq, p, enq_flags);
2676 } else {
2677 move_remote_task_to_local_dsq(p, enq_flags,
2678 src_rq, dst_rq);
2679 /* task has been moved to dst_rq, which is now locked */
2680 locked_rq = dst_rq;
2681 }
2682
2683 /* if the destination CPU is idle, wake it up */
2684 if (sched_class_above(p->sched_class, dst_rq->curr->sched_class))
2685 resched_curr(dst_rq);
2686 }
2687
2688 /* switch back to @rq lock */
2689 if (locked_rq != rq) {
2690 raw_spin_rq_unlock(locked_rq);
2691 raw_spin_rq_lock(rq);
2692 }
2693 #else /* CONFIG_SMP */
2694 BUG(); /* control can not reach here on UP */
2695 #endif /* CONFIG_SMP */
2696 }
2697
2698 /**
2699 * finish_dispatch - Asynchronously finish dispatching a task
2700 * @rq: current rq which is locked
2701 * @p: task to finish dispatching
2702 * @qseq_at_dispatch: qseq when @p started getting dispatched
2703 * @dsq_id: destination DSQ ID
2704 * @enq_flags: %SCX_ENQ_*
2705 *
2706 * Dispatching to local DSQs may need to wait for queueing to complete or
2707 * require rq lock dancing. As we don't wanna do either while inside
2708 * ops.dispatch() to avoid locking order inversion, we split dispatching into
2709 * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the
2710 * task and its qseq. Once ops.dispatch() returns, this function is called to
2711 * finish up.
2712 *
2713 * There is no guarantee that @p is still valid for dispatching or even that it
2714 * was valid in the first place. Make sure that the task is still owned by the
2715 * BPF scheduler and claim the ownership before dispatching.
2716 */
finish_dispatch(struct rq * rq,struct task_struct * p,unsigned long qseq_at_dispatch,u64 dsq_id,u64 enq_flags)2717 static void finish_dispatch(struct rq *rq, struct task_struct *p,
2718 unsigned long qseq_at_dispatch,
2719 u64 dsq_id, u64 enq_flags)
2720 {
2721 struct scx_dispatch_q *dsq;
2722 unsigned long opss;
2723
2724 touch_core_sched_dispatch(rq, p);
2725 retry:
2726 /*
2727 * No need for _acquire here. @p is accessed only after a successful
2728 * try_cmpxchg to DISPATCHING.
2729 */
2730 opss = atomic_long_read(&p->scx.ops_state);
2731
2732 switch (opss & SCX_OPSS_STATE_MASK) {
2733 case SCX_OPSS_DISPATCHING:
2734 case SCX_OPSS_NONE:
2735 /* someone else already got to it */
2736 return;
2737 case SCX_OPSS_QUEUED:
2738 /*
2739 * If qseq doesn't match, @p has gone through at least one
2740 * dispatch/dequeue and re-enqueue cycle between
2741 * scx_bpf_dsq_insert() and here and we have no claim on it.
2742 */
2743 if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch)
2744 return;
2745
2746 /*
2747 * While we know @p is accessible, we don't yet have a claim on
2748 * it - the BPF scheduler is allowed to dispatch tasks
2749 * spuriously and there can be a racing dequeue attempt. Let's
2750 * claim @p by atomically transitioning it from QUEUED to
2751 * DISPATCHING.
2752 */
2753 if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
2754 SCX_OPSS_DISPATCHING)))
2755 break;
2756 goto retry;
2757 case SCX_OPSS_QUEUEING:
2758 /*
2759 * do_enqueue_task() is in the process of transferring the task
2760 * to the BPF scheduler while holding @p's rq lock. As we aren't
2761 * holding any kernel or BPF resource that the enqueue path may
2762 * depend upon, it's safe to wait.
2763 */
2764 wait_ops_state(p, opss);
2765 goto retry;
2766 }
2767
2768 BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED));
2769
2770 dsq = find_dsq_for_dispatch(this_rq(), dsq_id, p);
2771
2772 if (dsq->id == SCX_DSQ_LOCAL)
2773 dispatch_to_local_dsq(rq, dsq, p, enq_flags);
2774 else
2775 dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
2776 }
2777
flush_dispatch_buf(struct rq * rq)2778 static void flush_dispatch_buf(struct rq *rq)
2779 {
2780 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
2781 u32 u;
2782
2783 for (u = 0; u < dspc->cursor; u++) {
2784 struct scx_dsp_buf_ent *ent = &dspc->buf[u];
2785
2786 finish_dispatch(rq, ent->task, ent->qseq, ent->dsq_id,
2787 ent->enq_flags);
2788 }
2789
2790 dspc->nr_tasks += dspc->cursor;
2791 dspc->cursor = 0;
2792 }
2793
balance_one(struct rq * rq,struct task_struct * prev)2794 static int balance_one(struct rq *rq, struct task_struct *prev)
2795 {
2796 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
2797 bool prev_on_scx = prev->sched_class == &ext_sched_class;
2798 bool prev_on_rq = prev->scx.flags & SCX_TASK_QUEUED;
2799 int nr_loops = SCX_DSP_MAX_LOOPS;
2800
2801 lockdep_assert_rq_held(rq);
2802 rq->scx.flags |= SCX_RQ_IN_BALANCE;
2803 rq->scx.flags &= ~(SCX_RQ_BAL_PENDING | SCX_RQ_BAL_KEEP);
2804
2805 if (static_branch_unlikely(&scx_ops_cpu_preempt) &&
2806 unlikely(rq->scx.cpu_released)) {
2807 /*
2808 * If the previous sched_class for the current CPU was not SCX,
2809 * notify the BPF scheduler that it again has control of the
2810 * core. This callback complements ->cpu_release(), which is
2811 * emitted in switch_class().
2812 */
2813 if (SCX_HAS_OP(cpu_acquire))
2814 SCX_CALL_OP(SCX_KF_REST, cpu_acquire, cpu_of(rq), NULL);
2815 rq->scx.cpu_released = false;
2816 }
2817
2818 if (prev_on_scx) {
2819 update_curr_scx(rq);
2820
2821 /*
2822 * If @prev is runnable & has slice left, it has priority and
2823 * fetching more just increases latency for the fetched tasks.
2824 * Tell pick_task_scx() to keep running @prev. If the BPF
2825 * scheduler wants to handle this explicitly, it should
2826 * implement ->cpu_release().
2827 *
2828 * See scx_ops_disable_workfn() for the explanation on the
2829 * bypassing test.
2830 */
2831 if (prev_on_rq && prev->scx.slice && !scx_rq_bypassing(rq)) {
2832 rq->scx.flags |= SCX_RQ_BAL_KEEP;
2833 goto has_tasks;
2834 }
2835 }
2836
2837 /* if there already are tasks to run, nothing to do */
2838 if (rq->scx.local_dsq.nr)
2839 goto has_tasks;
2840
2841 if (consume_global_dsq(rq))
2842 goto has_tasks;
2843
2844 if (!SCX_HAS_OP(dispatch) || scx_rq_bypassing(rq) || !scx_rq_online(rq))
2845 goto no_tasks;
2846
2847 dspc->rq = rq;
2848
2849 /*
2850 * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock,
2851 * the local DSQ might still end up empty after a successful
2852 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
2853 * produced some tasks, retry. The BPF scheduler may depend on this
2854 * looping behavior to simplify its implementation.
2855 */
2856 do {
2857 dspc->nr_tasks = 0;
2858
2859 SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq),
2860 prev_on_scx ? prev : NULL);
2861
2862 flush_dispatch_buf(rq);
2863
2864 if (prev_on_rq && prev->scx.slice) {
2865 rq->scx.flags |= SCX_RQ_BAL_KEEP;
2866 goto has_tasks;
2867 }
2868 if (rq->scx.local_dsq.nr)
2869 goto has_tasks;
2870 if (consume_global_dsq(rq))
2871 goto has_tasks;
2872
2873 /*
2874 * ops.dispatch() can trap us in this loop by repeatedly
2875 * dispatching ineligible tasks. Break out once in a while to
2876 * allow the watchdog to run. As IRQ can't be enabled in
2877 * balance(), we want to complete this scheduling cycle and then
2878 * start a new one. IOW, we want to call resched_curr() on the
2879 * next, most likely idle, task, not the current one. Use
2880 * scx_bpf_kick_cpu() for deferred kicking.
2881 */
2882 if (unlikely(!--nr_loops)) {
2883 scx_bpf_kick_cpu(cpu_of(rq), 0);
2884 break;
2885 }
2886 } while (dspc->nr_tasks);
2887
2888 no_tasks:
2889 /*
2890 * Didn't find another task to run. Keep running @prev unless
2891 * %SCX_OPS_ENQ_LAST is in effect.
2892 */
2893 if (prev_on_rq && (!static_branch_unlikely(&scx_ops_enq_last) ||
2894 scx_rq_bypassing(rq))) {
2895 rq->scx.flags |= SCX_RQ_BAL_KEEP;
2896 goto has_tasks;
2897 }
2898 rq->scx.flags &= ~SCX_RQ_IN_BALANCE;
2899 return false;
2900
2901 has_tasks:
2902 rq->scx.flags &= ~SCX_RQ_IN_BALANCE;
2903 return true;
2904 }
2905
balance_scx(struct rq * rq,struct task_struct * prev,struct rq_flags * rf)2906 static int balance_scx(struct rq *rq, struct task_struct *prev,
2907 struct rq_flags *rf)
2908 {
2909 int ret;
2910
2911 rq_unpin_lock(rq, rf);
2912
2913 ret = balance_one(rq, prev);
2914
2915 #ifdef CONFIG_SCHED_SMT
2916 /*
2917 * When core-sched is enabled, this ops.balance() call will be followed
2918 * by pick_task_scx() on this CPU and the SMT siblings. Balance the
2919 * siblings too.
2920 */
2921 if (sched_core_enabled(rq)) {
2922 const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq));
2923 int scpu;
2924
2925 for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) {
2926 struct rq *srq = cpu_rq(scpu);
2927 struct task_struct *sprev = srq->curr;
2928
2929 WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq));
2930 update_rq_clock(srq);
2931 balance_one(srq, sprev);
2932 }
2933 }
2934 #endif
2935 rq_repin_lock(rq, rf);
2936
2937 return ret;
2938 }
2939
process_ddsp_deferred_locals(struct rq * rq)2940 static void process_ddsp_deferred_locals(struct rq *rq)
2941 {
2942 struct task_struct *p;
2943
2944 lockdep_assert_rq_held(rq);
2945
2946 /*
2947 * Now that @rq can be unlocked, execute the deferred enqueueing of
2948 * tasks directly dispatched to the local DSQs of other CPUs. See
2949 * direct_dispatch(). Keep popping from the head instead of using
2950 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
2951 * temporarily.
2952 */
2953 while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
2954 struct task_struct, scx.dsq_list.node))) {
2955 struct scx_dispatch_q *dsq;
2956
2957 list_del_init(&p->scx.dsq_list.node);
2958
2959 dsq = find_dsq_for_dispatch(rq, p->scx.ddsp_dsq_id, p);
2960 if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
2961 dispatch_to_local_dsq(rq, dsq, p, p->scx.ddsp_enq_flags);
2962 }
2963 }
2964
set_next_task_scx(struct rq * rq,struct task_struct * p,bool first)2965 static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
2966 {
2967 if (p->scx.flags & SCX_TASK_QUEUED) {
2968 /*
2969 * Core-sched might decide to execute @p before it is
2970 * dispatched. Call ops_dequeue() to notify the BPF scheduler.
2971 */
2972 ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC);
2973 dispatch_dequeue(rq, p);
2974 }
2975
2976 p->se.exec_start = rq_clock_task(rq);
2977
2978 /* see dequeue_task_scx() on why we skip when !QUEUED */
2979 if (SCX_HAS_OP(running) && (p->scx.flags & SCX_TASK_QUEUED))
2980 SCX_CALL_OP_TASK(SCX_KF_REST, running, p);
2981
2982 clr_task_runnable(p, true);
2983
2984 /*
2985 * @p is getting newly scheduled or got kicked after someone updated its
2986 * slice. Refresh whether tick can be stopped. See scx_can_stop_tick().
2987 */
2988 if ((p->scx.slice == SCX_SLICE_INF) !=
2989 (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
2990 if (p->scx.slice == SCX_SLICE_INF)
2991 rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
2992 else
2993 rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
2994
2995 sched_update_tick_dependency(rq);
2996
2997 /*
2998 * For now, let's refresh the load_avgs just when transitioning
2999 * in and out of nohz. In the future, we might want to add a
3000 * mechanism which calls the following periodically on
3001 * tick-stopped CPUs.
3002 */
3003 update_other_load_avgs(rq);
3004 }
3005 }
3006
3007 static enum scx_cpu_preempt_reason
preempt_reason_from_class(const struct sched_class * class)3008 preempt_reason_from_class(const struct sched_class *class)
3009 {
3010 #ifdef CONFIG_SMP
3011 if (class == &stop_sched_class)
3012 return SCX_CPU_PREEMPT_STOP;
3013 #endif
3014 if (class == &dl_sched_class)
3015 return SCX_CPU_PREEMPT_DL;
3016 if (class == &rt_sched_class)
3017 return SCX_CPU_PREEMPT_RT;
3018 return SCX_CPU_PREEMPT_UNKNOWN;
3019 }
3020
switch_class(struct rq * rq,struct task_struct * next)3021 static void switch_class(struct rq *rq, struct task_struct *next)
3022 {
3023 const struct sched_class *next_class = next->sched_class;
3024
3025 #ifdef CONFIG_SMP
3026 /*
3027 * Pairs with the smp_load_acquire() issued by a CPU in
3028 * kick_cpus_irq_workfn() who is waiting for this CPU to perform a
3029 * resched.
3030 */
3031 smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1);
3032 #endif
3033 if (!static_branch_unlikely(&scx_ops_cpu_preempt))
3034 return;
3035
3036 /*
3037 * The callback is conceptually meant to convey that the CPU is no
3038 * longer under the control of SCX. Therefore, don't invoke the callback
3039 * if the next class is below SCX (in which case the BPF scheduler has
3040 * actively decided not to schedule any tasks on the CPU).
3041 */
3042 if (sched_class_above(&ext_sched_class, next_class))
3043 return;
3044
3045 /*
3046 * At this point we know that SCX was preempted by a higher priority
3047 * sched_class, so invoke the ->cpu_release() callback if we have not
3048 * done so already. We only send the callback once between SCX being
3049 * preempted, and it regaining control of the CPU.
3050 *
3051 * ->cpu_release() complements ->cpu_acquire(), which is emitted the
3052 * next time that balance_scx() is invoked.
3053 */
3054 if (!rq->scx.cpu_released) {
3055 if (SCX_HAS_OP(cpu_release)) {
3056 struct scx_cpu_release_args args = {
3057 .reason = preempt_reason_from_class(next_class),
3058 .task = next,
3059 };
3060
3061 SCX_CALL_OP(SCX_KF_CPU_RELEASE,
3062 cpu_release, cpu_of(rq), &args);
3063 }
3064 rq->scx.cpu_released = true;
3065 }
3066 }
3067
put_prev_task_scx(struct rq * rq,struct task_struct * p,struct task_struct * next)3068 static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
3069 struct task_struct *next)
3070 {
3071 update_curr_scx(rq);
3072
3073 /* see dequeue_task_scx() on why we skip when !QUEUED */
3074 if (SCX_HAS_OP(stopping) && (p->scx.flags & SCX_TASK_QUEUED))
3075 SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true);
3076
3077 if (p->scx.flags & SCX_TASK_QUEUED) {
3078 set_task_runnable(rq, p);
3079
3080 /*
3081 * If @p has slice left and is being put, @p is getting
3082 * preempted by a higher priority scheduler class or core-sched
3083 * forcing a different task. Leave it at the head of the local
3084 * DSQ.
3085 */
3086 if (p->scx.slice && !scx_rq_bypassing(rq)) {
3087 dispatch_enqueue(&rq->scx.local_dsq, p, SCX_ENQ_HEAD);
3088 goto switch_class;
3089 }
3090
3091 /*
3092 * If @p is runnable but we're about to enter a lower
3093 * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
3094 * ops.enqueue() that @p is the only one available for this cpu,
3095 * which should trigger an explicit follow-up scheduling event.
3096 */
3097 if (sched_class_above(&ext_sched_class, next->sched_class)) {
3098 WARN_ON_ONCE(!static_branch_unlikely(&scx_ops_enq_last));
3099 do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
3100 } else {
3101 do_enqueue_task(rq, p, 0, -1);
3102 }
3103 }
3104
3105 switch_class:
3106 if (next && next->sched_class != &ext_sched_class)
3107 switch_class(rq, next);
3108 }
3109
first_local_task(struct rq * rq)3110 static struct task_struct *first_local_task(struct rq *rq)
3111 {
3112 return list_first_entry_or_null(&rq->scx.local_dsq.list,
3113 struct task_struct, scx.dsq_list.node);
3114 }
3115
pick_task_scx(struct rq * rq)3116 static struct task_struct *pick_task_scx(struct rq *rq)
3117 {
3118 struct task_struct *prev = rq->curr;
3119 struct task_struct *p;
3120 bool keep_prev = rq->scx.flags & SCX_RQ_BAL_KEEP;
3121 bool kick_idle = false;
3122
3123 /*
3124 * WORKAROUND:
3125 *
3126 * %SCX_RQ_BAL_KEEP should be set iff $prev is on SCX as it must just
3127 * have gone through balance_scx(). Unfortunately, there currently is a
3128 * bug where fair could say yes on balance() but no on pick_task(),
3129 * which then ends up calling pick_task_scx() without preceding
3130 * balance_scx().
3131 *
3132 * Keep running @prev if possible and avoid stalling from entering idle
3133 * without balancing.
3134 *
3135 * Once fair is fixed, remove the workaround and trigger WARN_ON_ONCE()
3136 * if pick_task_scx() is called without preceding balance_scx().
3137 */
3138 if (unlikely(rq->scx.flags & SCX_RQ_BAL_PENDING)) {
3139 if (prev->scx.flags & SCX_TASK_QUEUED) {
3140 keep_prev = true;
3141 } else {
3142 keep_prev = false;
3143 kick_idle = true;
3144 }
3145 } else if (unlikely(keep_prev &&
3146 prev->sched_class != &ext_sched_class)) {
3147 /*
3148 * Can happen while enabling as SCX_RQ_BAL_PENDING assertion is
3149 * conditional on scx_enabled() and may have been skipped.
3150 */
3151 WARN_ON_ONCE(scx_ops_enable_state() == SCX_OPS_ENABLED);
3152 keep_prev = false;
3153 }
3154
3155 /*
3156 * If balance_scx() is telling us to keep running @prev, replenish slice
3157 * if necessary and keep running @prev. Otherwise, pop the first one
3158 * from the local DSQ.
3159 */
3160 if (keep_prev) {
3161 p = prev;
3162 if (!p->scx.slice)
3163 p->scx.slice = SCX_SLICE_DFL;
3164 } else {
3165 p = first_local_task(rq);
3166 if (!p) {
3167 if (kick_idle)
3168 scx_bpf_kick_cpu(cpu_of(rq), SCX_KICK_IDLE);
3169 return NULL;
3170 }
3171
3172 if (unlikely(!p->scx.slice)) {
3173 if (!scx_rq_bypassing(rq) && !scx_warned_zero_slice) {
3174 printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n",
3175 p->comm, p->pid, __func__);
3176 scx_warned_zero_slice = true;
3177 }
3178 p->scx.slice = SCX_SLICE_DFL;
3179 }
3180 }
3181
3182 return p;
3183 }
3184
3185 #ifdef CONFIG_SCHED_CORE
3186 /**
3187 * scx_prio_less - Task ordering for core-sched
3188 * @a: task A
3189 * @b: task B
3190 * @in_fi: in forced idle state
3191 *
3192 * Core-sched is implemented as an additional scheduling layer on top of the
3193 * usual sched_class'es and needs to find out the expected task ordering. For
3194 * SCX, core-sched calls this function to interrogate the task ordering.
3195 *
3196 * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used
3197 * to implement the default task ordering. The older the timestamp, the higher
3198 * priority the task - the global FIFO ordering matching the default scheduling
3199 * behavior.
3200 *
3201 * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to
3202 * implement FIFO ordering within each local DSQ. See pick_task_scx().
3203 */
scx_prio_less(const struct task_struct * a,const struct task_struct * b,bool in_fi)3204 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
3205 bool in_fi)
3206 {
3207 /*
3208 * The const qualifiers are dropped from task_struct pointers when
3209 * calling ops.core_sched_before(). Accesses are controlled by the
3210 * verifier.
3211 */
3212 if (SCX_HAS_OP(core_sched_before) && !scx_rq_bypassing(task_rq(a)))
3213 return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before,
3214 (struct task_struct *)a,
3215 (struct task_struct *)b);
3216 else
3217 return time_after64(a->scx.core_sched_at, b->scx.core_sched_at);
3218 }
3219 #endif /* CONFIG_SCHED_CORE */
3220
3221 #ifdef CONFIG_SMP
3222
test_and_clear_cpu_idle(int cpu)3223 static bool test_and_clear_cpu_idle(int cpu)
3224 {
3225 #ifdef CONFIG_SCHED_SMT
3226 /*
3227 * SMT mask should be cleared whether we can claim @cpu or not. The SMT
3228 * cluster is not wholly idle either way. This also prevents
3229 * scx_pick_idle_cpu() from getting caught in an infinite loop.
3230 */
3231 if (sched_smt_active()) {
3232 const struct cpumask *smt = cpu_smt_mask(cpu);
3233
3234 /*
3235 * If offline, @cpu is not its own sibling and
3236 * scx_pick_idle_cpu() can get caught in an infinite loop as
3237 * @cpu is never cleared from idle_masks.smt. Ensure that @cpu
3238 * is eventually cleared.
3239 *
3240 * NOTE: Use cpumask_intersects() and cpumask_test_cpu() to
3241 * reduce memory writes, which may help alleviate cache
3242 * coherence pressure.
3243 */
3244 if (cpumask_intersects(smt, idle_masks.smt))
3245 cpumask_andnot(idle_masks.smt, idle_masks.smt, smt);
3246 else if (cpumask_test_cpu(cpu, idle_masks.smt))
3247 __cpumask_clear_cpu(cpu, idle_masks.smt);
3248 }
3249 #endif
3250 return cpumask_test_and_clear_cpu(cpu, idle_masks.cpu);
3251 }
3252
scx_pick_idle_cpu(const struct cpumask * cpus_allowed,u64 flags)3253 static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags)
3254 {
3255 int cpu;
3256
3257 retry:
3258 if (sched_smt_active()) {
3259 cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed);
3260 if (cpu < nr_cpu_ids)
3261 goto found;
3262
3263 if (flags & SCX_PICK_IDLE_CORE)
3264 return -EBUSY;
3265 }
3266
3267 cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed);
3268 if (cpu >= nr_cpu_ids)
3269 return -EBUSY;
3270
3271 found:
3272 if (test_and_clear_cpu_idle(cpu))
3273 return cpu;
3274 else
3275 goto retry;
3276 }
3277
3278 /*
3279 * Return the amount of CPUs in the same LLC domain of @cpu (or zero if the LLC
3280 * domain is not defined).
3281 */
llc_weight(s32 cpu)3282 static unsigned int llc_weight(s32 cpu)
3283 {
3284 struct sched_domain *sd;
3285
3286 sd = rcu_dereference(per_cpu(sd_llc, cpu));
3287 if (!sd)
3288 return 0;
3289
3290 return sd->span_weight;
3291 }
3292
3293 /*
3294 * Return the cpumask representing the LLC domain of @cpu (or NULL if the LLC
3295 * domain is not defined).
3296 */
llc_span(s32 cpu)3297 static struct cpumask *llc_span(s32 cpu)
3298 {
3299 struct sched_domain *sd;
3300
3301 sd = rcu_dereference(per_cpu(sd_llc, cpu));
3302 if (!sd)
3303 return 0;
3304
3305 return sched_domain_span(sd);
3306 }
3307
3308 /*
3309 * Return the amount of CPUs in the same NUMA domain of @cpu (or zero if the
3310 * NUMA domain is not defined).
3311 */
numa_weight(s32 cpu)3312 static unsigned int numa_weight(s32 cpu)
3313 {
3314 struct sched_domain *sd;
3315 struct sched_group *sg;
3316
3317 sd = rcu_dereference(per_cpu(sd_numa, cpu));
3318 if (!sd)
3319 return 0;
3320 sg = sd->groups;
3321 if (!sg)
3322 return 0;
3323
3324 return sg->group_weight;
3325 }
3326
3327 /*
3328 * Return the cpumask representing the NUMA domain of @cpu (or NULL if the NUMA
3329 * domain is not defined).
3330 */
numa_span(s32 cpu)3331 static struct cpumask *numa_span(s32 cpu)
3332 {
3333 struct sched_domain *sd;
3334 struct sched_group *sg;
3335
3336 sd = rcu_dereference(per_cpu(sd_numa, cpu));
3337 if (!sd)
3338 return NULL;
3339 sg = sd->groups;
3340 if (!sg)
3341 return NULL;
3342
3343 return sched_group_span(sg);
3344 }
3345
3346 /*
3347 * Return true if the LLC domains do not perfectly overlap with the NUMA
3348 * domains, false otherwise.
3349 */
llc_numa_mismatch(void)3350 static bool llc_numa_mismatch(void)
3351 {
3352 int cpu;
3353
3354 /*
3355 * We need to scan all online CPUs to verify whether their scheduling
3356 * domains overlap.
3357 *
3358 * While it is rare to encounter architectures with asymmetric NUMA
3359 * topologies, CPU hotplugging or virtualized environments can result
3360 * in asymmetric configurations.
3361 *
3362 * For example:
3363 *
3364 * NUMA 0:
3365 * - LLC 0: cpu0..cpu7
3366 * - LLC 1: cpu8..cpu15 [offline]
3367 *
3368 * NUMA 1:
3369 * - LLC 0: cpu16..cpu23
3370 * - LLC 1: cpu24..cpu31
3371 *
3372 * In this case, if we only check the first online CPU (cpu0), we might
3373 * incorrectly assume that the LLC and NUMA domains are fully
3374 * overlapping, which is incorrect (as NUMA 1 has two distinct LLC
3375 * domains).
3376 */
3377 for_each_online_cpu(cpu)
3378 if (llc_weight(cpu) != numa_weight(cpu))
3379 return true;
3380
3381 return false;
3382 }
3383
3384 /*
3385 * Initialize topology-aware scheduling.
3386 *
3387 * Detect if the system has multiple LLC or multiple NUMA domains and enable
3388 * cache-aware / NUMA-aware scheduling optimizations in the default CPU idle
3389 * selection policy.
3390 *
3391 * Assumption: the kernel's internal topology representation assumes that each
3392 * CPU belongs to a single LLC domain, and that each LLC domain is entirely
3393 * contained within a single NUMA node.
3394 */
update_selcpu_topology(void)3395 static void update_selcpu_topology(void)
3396 {
3397 bool enable_llc = false, enable_numa = false;
3398 unsigned int nr_cpus;
3399 s32 cpu = cpumask_first(cpu_online_mask);
3400
3401 /*
3402 * Enable LLC domain optimization only when there are multiple LLC
3403 * domains among the online CPUs. If all online CPUs are part of a
3404 * single LLC domain, the idle CPU selection logic can choose any
3405 * online CPU without bias.
3406 *
3407 * Note that it is sufficient to check the LLC domain of the first
3408 * online CPU to determine whether a single LLC domain includes all
3409 * CPUs.
3410 */
3411 rcu_read_lock();
3412 nr_cpus = llc_weight(cpu);
3413 if (nr_cpus > 0) {
3414 if (nr_cpus < num_online_cpus())
3415 enable_llc = true;
3416 pr_debug("sched_ext: LLC=%*pb weight=%u\n",
3417 cpumask_pr_args(llc_span(cpu)), llc_weight(cpu));
3418 }
3419
3420 /*
3421 * Enable NUMA optimization only when there are multiple NUMA domains
3422 * among the online CPUs and the NUMA domains don't perfectly overlaps
3423 * with the LLC domains.
3424 *
3425 * If all CPUs belong to the same NUMA node and the same LLC domain,
3426 * enabling both NUMA and LLC optimizations is unnecessary, as checking
3427 * for an idle CPU in the same domain twice is redundant.
3428 */
3429 nr_cpus = numa_weight(cpu);
3430 if (nr_cpus > 0) {
3431 if (nr_cpus < num_online_cpus() && llc_numa_mismatch())
3432 enable_numa = true;
3433 pr_debug("sched_ext: NUMA=%*pb weight=%u\n",
3434 cpumask_pr_args(numa_span(cpu)), numa_weight(cpu));
3435 }
3436 rcu_read_unlock();
3437
3438 pr_debug("sched_ext: LLC idle selection %s\n",
3439 str_enabled_disabled(enable_llc));
3440 pr_debug("sched_ext: NUMA idle selection %s\n",
3441 str_enabled_disabled(enable_numa));
3442
3443 if (enable_llc)
3444 static_branch_enable_cpuslocked(&scx_selcpu_topo_llc);
3445 else
3446 static_branch_disable_cpuslocked(&scx_selcpu_topo_llc);
3447 if (enable_numa)
3448 static_branch_enable_cpuslocked(&scx_selcpu_topo_numa);
3449 else
3450 static_branch_disable_cpuslocked(&scx_selcpu_topo_numa);
3451 }
3452
3453 /*
3454 * Built-in CPU idle selection policy:
3455 *
3456 * 1. Prioritize full-idle cores:
3457 * - always prioritize CPUs from fully idle cores (both logical CPUs are
3458 * idle) to avoid interference caused by SMT.
3459 *
3460 * 2. Reuse the same CPU:
3461 * - prefer the last used CPU to take advantage of cached data (L1, L2) and
3462 * branch prediction optimizations.
3463 *
3464 * 3. Pick a CPU within the same LLC (Last-Level Cache):
3465 * - if the above conditions aren't met, pick a CPU that shares the same LLC
3466 * to maintain cache locality.
3467 *
3468 * 4. Pick a CPU within the same NUMA node, if enabled:
3469 * - choose a CPU from the same NUMA node to reduce memory access latency.
3470 *
3471 * 5. Pick any idle CPU usable by the task.
3472 *
3473 * Step 3 and 4 are performed only if the system has, respectively, multiple
3474 * LLC domains / multiple NUMA nodes (see scx_selcpu_topo_llc and
3475 * scx_selcpu_topo_numa).
3476 *
3477 * NOTE: tasks that can only run on 1 CPU are excluded by this logic, because
3478 * we never call ops.select_cpu() for them, see select_task_rq().
3479 */
scx_select_cpu_dfl(struct task_struct * p,s32 prev_cpu,u64 wake_flags,bool * found)3480 static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
3481 u64 wake_flags, bool *found)
3482 {
3483 const struct cpumask *llc_cpus = NULL;
3484 const struct cpumask *numa_cpus = NULL;
3485 s32 cpu;
3486
3487 *found = false;
3488
3489 /*
3490 * This is necessary to protect llc_cpus.
3491 */
3492 rcu_read_lock();
3493
3494 /*
3495 * Determine the scheduling domain only if the task is allowed to run
3496 * on all CPUs.
3497 *
3498 * This is done primarily for efficiency, as it avoids the overhead of
3499 * updating a cpumask every time we need to select an idle CPU (which
3500 * can be costly in large SMP systems), but it also aligns logically:
3501 * if a task's scheduling domain is restricted by user-space (through
3502 * CPU affinity), the task will simply use the flat scheduling domain
3503 * defined by user-space.
3504 */
3505 if (p->nr_cpus_allowed >= num_possible_cpus()) {
3506 if (static_branch_maybe(CONFIG_NUMA, &scx_selcpu_topo_numa))
3507 numa_cpus = numa_span(prev_cpu);
3508
3509 if (static_branch_maybe(CONFIG_SCHED_MC, &scx_selcpu_topo_llc))
3510 llc_cpus = llc_span(prev_cpu);
3511 }
3512
3513 /*
3514 * If WAKE_SYNC, try to migrate the wakee to the waker's CPU.
3515 */
3516 if (wake_flags & SCX_WAKE_SYNC) {
3517 cpu = smp_processor_id();
3518
3519 /*
3520 * If the waker's CPU is cache affine and prev_cpu is idle,
3521 * then avoid a migration.
3522 */
3523 if (cpus_share_cache(cpu, prev_cpu) &&
3524 test_and_clear_cpu_idle(prev_cpu)) {
3525 cpu = prev_cpu;
3526 goto cpu_found;
3527 }
3528
3529 /*
3530 * If the waker's local DSQ is empty, and the system is under
3531 * utilized, try to wake up @p to the local DSQ of the waker.
3532 *
3533 * Checking only for an empty local DSQ is insufficient as it
3534 * could give the wakee an unfair advantage when the system is
3535 * oversaturated.
3536 *
3537 * Checking only for the presence of idle CPUs is also
3538 * insufficient as the local DSQ of the waker could have tasks
3539 * piled up on it even if there is an idle core elsewhere on
3540 * the system.
3541 */
3542 if (!cpumask_empty(idle_masks.cpu) &&
3543 !(current->flags & PF_EXITING) &&
3544 cpu_rq(cpu)->scx.local_dsq.nr == 0) {
3545 if (cpumask_test_cpu(cpu, p->cpus_ptr))
3546 goto cpu_found;
3547 }
3548 }
3549
3550 /*
3551 * If CPU has SMT, any wholly idle CPU is likely a better pick than
3552 * partially idle @prev_cpu.
3553 */
3554 if (sched_smt_active()) {
3555 /*
3556 * Keep using @prev_cpu if it's part of a fully idle core.
3557 */
3558 if (cpumask_test_cpu(prev_cpu, idle_masks.smt) &&
3559 test_and_clear_cpu_idle(prev_cpu)) {
3560 cpu = prev_cpu;
3561 goto cpu_found;
3562 }
3563
3564 /*
3565 * Search for any fully idle core in the same LLC domain.
3566 */
3567 if (llc_cpus) {
3568 cpu = scx_pick_idle_cpu(llc_cpus, SCX_PICK_IDLE_CORE);
3569 if (cpu >= 0)
3570 goto cpu_found;
3571 }
3572
3573 /*
3574 * Search for any fully idle core in the same NUMA node.
3575 */
3576 if (numa_cpus) {
3577 cpu = scx_pick_idle_cpu(numa_cpus, SCX_PICK_IDLE_CORE);
3578 if (cpu >= 0)
3579 goto cpu_found;
3580 }
3581
3582 /*
3583 * Search for any full idle core usable by the task.
3584 */
3585 cpu = scx_pick_idle_cpu(p->cpus_ptr, SCX_PICK_IDLE_CORE);
3586 if (cpu >= 0)
3587 goto cpu_found;
3588 }
3589
3590 /*
3591 * Use @prev_cpu if it's idle.
3592 */
3593 if (test_and_clear_cpu_idle(prev_cpu)) {
3594 cpu = prev_cpu;
3595 goto cpu_found;
3596 }
3597
3598 /*
3599 * Search for any idle CPU in the same LLC domain.
3600 */
3601 if (llc_cpus) {
3602 cpu = scx_pick_idle_cpu(llc_cpus, 0);
3603 if (cpu >= 0)
3604 goto cpu_found;
3605 }
3606
3607 /*
3608 * Search for any idle CPU in the same NUMA node.
3609 */
3610 if (numa_cpus) {
3611 cpu = scx_pick_idle_cpu(numa_cpus, 0);
3612 if (cpu >= 0)
3613 goto cpu_found;
3614 }
3615
3616 /*
3617 * Search for any idle CPU usable by the task.
3618 */
3619 cpu = scx_pick_idle_cpu(p->cpus_ptr, 0);
3620 if (cpu >= 0)
3621 goto cpu_found;
3622
3623 rcu_read_unlock();
3624 return prev_cpu;
3625
3626 cpu_found:
3627 rcu_read_unlock();
3628
3629 *found = true;
3630 return cpu;
3631 }
3632
select_task_rq_scx(struct task_struct * p,int prev_cpu,int wake_flags)3633 static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)
3634 {
3635 /*
3636 * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it
3637 * can be a good migration opportunity with low cache and memory
3638 * footprint. Returning a CPU different than @prev_cpu triggers
3639 * immediate rq migration. However, for SCX, as the current rq
3640 * association doesn't dictate where the task is going to run, this
3641 * doesn't fit well. If necessary, we can later add a dedicated method
3642 * which can decide to preempt self to force it through the regular
3643 * scheduling path.
3644 */
3645 if (unlikely(wake_flags & WF_EXEC))
3646 return prev_cpu;
3647
3648 if (SCX_HAS_OP(select_cpu) && !scx_rq_bypassing(task_rq(p))) {
3649 s32 cpu;
3650 struct task_struct **ddsp_taskp;
3651
3652 ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
3653 WARN_ON_ONCE(*ddsp_taskp);
3654 *ddsp_taskp = p;
3655
3656 cpu = SCX_CALL_OP_TASK_RET(SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU,
3657 select_cpu, p, prev_cpu, wake_flags);
3658 *ddsp_taskp = NULL;
3659 if (ops_cpu_valid(cpu, "from ops.select_cpu()"))
3660 return cpu;
3661 else
3662 return prev_cpu;
3663 } else {
3664 bool found;
3665 s32 cpu;
3666
3667 cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, &found);
3668 if (found) {
3669 p->scx.slice = SCX_SLICE_DFL;
3670 p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;
3671 }
3672 return cpu;
3673 }
3674 }
3675
task_woken_scx(struct rq * rq,struct task_struct * p)3676 static void task_woken_scx(struct rq *rq, struct task_struct *p)
3677 {
3678 run_deferred(rq);
3679 }
3680
set_cpus_allowed_scx(struct task_struct * p,struct affinity_context * ac)3681 static void set_cpus_allowed_scx(struct task_struct *p,
3682 struct affinity_context *ac)
3683 {
3684 set_cpus_allowed_common(p, ac);
3685
3686 /*
3687 * The effective cpumask is stored in @p->cpus_ptr which may temporarily
3688 * differ from the configured one in @p->cpus_mask. Always tell the bpf
3689 * scheduler the effective one.
3690 *
3691 * Fine-grained memory write control is enforced by BPF making the const
3692 * designation pointless. Cast it away when calling the operation.
3693 */
3694 if (SCX_HAS_OP(set_cpumask))
3695 SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p,
3696 (struct cpumask *)p->cpus_ptr);
3697 }
3698
reset_idle_masks(void)3699 static void reset_idle_masks(void)
3700 {
3701 /*
3702 * Consider all online cpus idle. Should converge to the actual state
3703 * quickly.
3704 */
3705 cpumask_copy(idle_masks.cpu, cpu_online_mask);
3706 cpumask_copy(idle_masks.smt, cpu_online_mask);
3707 }
3708
update_builtin_idle(int cpu,bool idle)3709 static void update_builtin_idle(int cpu, bool idle)
3710 {
3711 assign_cpu(cpu, idle_masks.cpu, idle);
3712
3713 #ifdef CONFIG_SCHED_SMT
3714 if (sched_smt_active()) {
3715 const struct cpumask *smt = cpu_smt_mask(cpu);
3716
3717 if (idle) {
3718 /*
3719 * idle_masks.smt handling is racy but that's fine as
3720 * it's only for optimization and self-correcting.
3721 */
3722 if (!cpumask_subset(smt, idle_masks.cpu))
3723 return;
3724 cpumask_or(idle_masks.smt, idle_masks.smt, smt);
3725 } else {
3726 cpumask_andnot(idle_masks.smt, idle_masks.smt, smt);
3727 }
3728 }
3729 #endif
3730 }
3731
3732 /*
3733 * Update the idle state of a CPU to @idle.
3734 *
3735 * If @do_notify is true, ops.update_idle() is invoked to notify the scx
3736 * scheduler of an actual idle state transition (idle to busy or vice
3737 * versa). If @do_notify is false, only the idle state in the idle masks is
3738 * refreshed without invoking ops.update_idle().
3739 *
3740 * This distinction is necessary, because an idle CPU can be "reserved" and
3741 * awakened via scx_bpf_pick_idle_cpu() + scx_bpf_kick_cpu(), marking it as
3742 * busy even if no tasks are dispatched. In this case, the CPU may return
3743 * to idle without a true state transition. Refreshing the idle masks
3744 * without invoking ops.update_idle() ensures accurate idle state tracking
3745 * while avoiding unnecessary updates and maintaining balanced state
3746 * transitions.
3747 */
__scx_update_idle(struct rq * rq,bool idle,bool do_notify)3748 void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)
3749 {
3750 int cpu = cpu_of(rq);
3751
3752 lockdep_assert_rq_held(rq);
3753
3754 /*
3755 * Trigger ops.update_idle() only when transitioning from a task to
3756 * the idle thread and vice versa.
3757 *
3758 * Idle transitions are indicated by do_notify being set to true,
3759 * managed by put_prev_task_idle()/set_next_task_idle().
3760 */
3761 if (SCX_HAS_OP(update_idle) && do_notify && !scx_rq_bypassing(rq))
3762 SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle);
3763
3764 /*
3765 * Update the idle masks:
3766 * - for real idle transitions (do_notify == true)
3767 * - for idle-to-idle transitions (indicated by the previous task
3768 * being the idle thread, managed by pick_task_idle())
3769 *
3770 * Skip updating idle masks if the previous task is not the idle
3771 * thread, since set_next_task_idle() has already handled it when
3772 * transitioning from a task to the idle thread (calling this
3773 * function with do_notify == true).
3774 *
3775 * In this way we can avoid updating the idle masks twice,
3776 * unnecessarily.
3777 */
3778 if (static_branch_likely(&scx_builtin_idle_enabled))
3779 if (do_notify || is_idle_task(rq->curr))
3780 update_builtin_idle(cpu, idle);
3781 }
3782
handle_hotplug(struct rq * rq,bool online)3783 static void handle_hotplug(struct rq *rq, bool online)
3784 {
3785 int cpu = cpu_of(rq);
3786
3787 atomic_long_inc(&scx_hotplug_seq);
3788
3789 if (scx_enabled())
3790 update_selcpu_topology();
3791
3792 if (online && SCX_HAS_OP(cpu_online))
3793 SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_online, cpu);
3794 else if (!online && SCX_HAS_OP(cpu_offline))
3795 SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_offline, cpu);
3796 else
3797 scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
3798 "cpu %d going %s, exiting scheduler", cpu,
3799 online ? "online" : "offline");
3800 }
3801
scx_rq_activate(struct rq * rq)3802 void scx_rq_activate(struct rq *rq)
3803 {
3804 handle_hotplug(rq, true);
3805 }
3806
scx_rq_deactivate(struct rq * rq)3807 void scx_rq_deactivate(struct rq *rq)
3808 {
3809 handle_hotplug(rq, false);
3810 }
3811
rq_online_scx(struct rq * rq)3812 static void rq_online_scx(struct rq *rq)
3813 {
3814 rq->scx.flags |= SCX_RQ_ONLINE;
3815 }
3816
rq_offline_scx(struct rq * rq)3817 static void rq_offline_scx(struct rq *rq)
3818 {
3819 rq->scx.flags &= ~SCX_RQ_ONLINE;
3820 }
3821
3822 #else /* CONFIG_SMP */
3823
test_and_clear_cpu_idle(int cpu)3824 static bool test_and_clear_cpu_idle(int cpu) { return false; }
scx_pick_idle_cpu(const struct cpumask * cpus_allowed,u64 flags)3825 static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags) { return -EBUSY; }
reset_idle_masks(void)3826 static void reset_idle_masks(void) {}
3827
3828 #endif /* CONFIG_SMP */
3829
check_rq_for_timeouts(struct rq * rq)3830 static bool check_rq_for_timeouts(struct rq *rq)
3831 {
3832 struct task_struct *p;
3833 struct rq_flags rf;
3834 bool timed_out = false;
3835
3836 rq_lock_irqsave(rq, &rf);
3837 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) {
3838 unsigned long last_runnable = p->scx.runnable_at;
3839
3840 if (unlikely(time_after(jiffies,
3841 last_runnable + scx_watchdog_timeout))) {
3842 u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
3843
3844 scx_ops_error_kind(SCX_EXIT_ERROR_STALL,
3845 "%s[%d] failed to run for %u.%03us",
3846 p->comm, p->pid,
3847 dur_ms / 1000, dur_ms % 1000);
3848 timed_out = true;
3849 break;
3850 }
3851 }
3852 rq_unlock_irqrestore(rq, &rf);
3853
3854 return timed_out;
3855 }
3856
scx_watchdog_workfn(struct work_struct * work)3857 static void scx_watchdog_workfn(struct work_struct *work)
3858 {
3859 int cpu;
3860
3861 WRITE_ONCE(scx_watchdog_timestamp, jiffies);
3862
3863 for_each_online_cpu(cpu) {
3864 if (unlikely(check_rq_for_timeouts(cpu_rq(cpu))))
3865 break;
3866
3867 cond_resched();
3868 }
3869 queue_delayed_work(system_unbound_wq, to_delayed_work(work),
3870 scx_watchdog_timeout / 2);
3871 }
3872
scx_tick(struct rq * rq)3873 void scx_tick(struct rq *rq)
3874 {
3875 unsigned long last_check;
3876
3877 if (!scx_enabled())
3878 return;
3879
3880 last_check = READ_ONCE(scx_watchdog_timestamp);
3881 if (unlikely(time_after(jiffies,
3882 last_check + READ_ONCE(scx_watchdog_timeout)))) {
3883 u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
3884
3885 scx_ops_error_kind(SCX_EXIT_ERROR_STALL,
3886 "watchdog failed to check in for %u.%03us",
3887 dur_ms / 1000, dur_ms % 1000);
3888 }
3889
3890 update_other_load_avgs(rq);
3891 }
3892
task_tick_scx(struct rq * rq,struct task_struct * curr,int queued)3893 static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
3894 {
3895 update_curr_scx(rq);
3896
3897 /*
3898 * While disabling, always resched and refresh core-sched timestamp as
3899 * we can't trust the slice management or ops.core_sched_before().
3900 */
3901 if (scx_rq_bypassing(rq)) {
3902 curr->scx.slice = 0;
3903 touch_core_sched(rq, curr);
3904 } else if (SCX_HAS_OP(tick)) {
3905 SCX_CALL_OP_TASK(SCX_KF_REST, tick, curr);
3906 }
3907
3908 if (!curr->scx.slice)
3909 resched_curr(rq);
3910 }
3911
3912 #ifdef CONFIG_EXT_GROUP_SCHED
tg_cgrp(struct task_group * tg)3913 static struct cgroup *tg_cgrp(struct task_group *tg)
3914 {
3915 /*
3916 * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup,
3917 * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the
3918 * root cgroup.
3919 */
3920 if (tg && tg->css.cgroup)
3921 return tg->css.cgroup;
3922 else
3923 return &cgrp_dfl_root.cgrp;
3924 }
3925
3926 #define SCX_INIT_TASK_ARGS_CGROUP(tg) .cgroup = tg_cgrp(tg),
3927
3928 #else /* CONFIG_EXT_GROUP_SCHED */
3929
3930 #define SCX_INIT_TASK_ARGS_CGROUP(tg)
3931
3932 #endif /* CONFIG_EXT_GROUP_SCHED */
3933
scx_get_task_state(const struct task_struct * p)3934 static enum scx_task_state scx_get_task_state(const struct task_struct *p)
3935 {
3936 return (p->scx.flags & SCX_TASK_STATE_MASK) >> SCX_TASK_STATE_SHIFT;
3937 }
3938
scx_set_task_state(struct task_struct * p,enum scx_task_state state)3939 static void scx_set_task_state(struct task_struct *p, enum scx_task_state state)
3940 {
3941 enum scx_task_state prev_state = scx_get_task_state(p);
3942 bool warn = false;
3943
3944 BUILD_BUG_ON(SCX_TASK_NR_STATES > (1 << SCX_TASK_STATE_BITS));
3945
3946 switch (state) {
3947 case SCX_TASK_NONE:
3948 break;
3949 case SCX_TASK_INIT:
3950 warn = prev_state != SCX_TASK_NONE;
3951 break;
3952 case SCX_TASK_READY:
3953 warn = prev_state == SCX_TASK_NONE;
3954 break;
3955 case SCX_TASK_ENABLED:
3956 warn = prev_state != SCX_TASK_READY;
3957 break;
3958 default:
3959 warn = true;
3960 return;
3961 }
3962
3963 WARN_ONCE(warn, "sched_ext: Invalid task state transition %d -> %d for %s[%d]",
3964 prev_state, state, p->comm, p->pid);
3965
3966 p->scx.flags &= ~SCX_TASK_STATE_MASK;
3967 p->scx.flags |= state << SCX_TASK_STATE_SHIFT;
3968 }
3969
scx_ops_init_task(struct task_struct * p,struct task_group * tg,bool fork)3970 static int scx_ops_init_task(struct task_struct *p, struct task_group *tg, bool fork)
3971 {
3972 int ret;
3973
3974 p->scx.disallow = false;
3975
3976 if (SCX_HAS_OP(init_task)) {
3977 struct scx_init_task_args args = {
3978 SCX_INIT_TASK_ARGS_CGROUP(tg)
3979 .fork = fork,
3980 };
3981
3982 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init_task, p, &args);
3983 if (unlikely(ret)) {
3984 ret = ops_sanitize_err("init_task", ret);
3985 return ret;
3986 }
3987 }
3988
3989 scx_set_task_state(p, SCX_TASK_INIT);
3990
3991 if (p->scx.disallow) {
3992 if (!fork) {
3993 struct rq *rq;
3994 struct rq_flags rf;
3995
3996 rq = task_rq_lock(p, &rf);
3997
3998 /*
3999 * We're in the load path and @p->policy will be applied
4000 * right after. Reverting @p->policy here and rejecting
4001 * %SCHED_EXT transitions from scx_check_setscheduler()
4002 * guarantees that if ops.init_task() sets @p->disallow,
4003 * @p can never be in SCX.
4004 */
4005 if (p->policy == SCHED_EXT) {
4006 p->policy = SCHED_NORMAL;
4007 atomic_long_inc(&scx_nr_rejected);
4008 }
4009
4010 task_rq_unlock(rq, p, &rf);
4011 } else if (p->policy == SCHED_EXT) {
4012 scx_ops_error("ops.init_task() set task->scx.disallow for %s[%d] during fork",
4013 p->comm, p->pid);
4014 }
4015 }
4016
4017 p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
4018 return 0;
4019 }
4020
scx_ops_enable_task(struct task_struct * p)4021 static void scx_ops_enable_task(struct task_struct *p)
4022 {
4023 u32 weight;
4024
4025 lockdep_assert_rq_held(task_rq(p));
4026
4027 /*
4028 * Set the weight before calling ops.enable() so that the scheduler
4029 * doesn't see a stale value if they inspect the task struct.
4030 */
4031 if (task_has_idle_policy(p))
4032 weight = WEIGHT_IDLEPRIO;
4033 else
4034 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
4035
4036 p->scx.weight = sched_weight_to_cgroup(weight);
4037
4038 if (SCX_HAS_OP(enable))
4039 SCX_CALL_OP_TASK(SCX_KF_REST, enable, p);
4040 scx_set_task_state(p, SCX_TASK_ENABLED);
4041
4042 if (SCX_HAS_OP(set_weight))
4043 SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx.weight);
4044 }
4045
scx_ops_disable_task(struct task_struct * p)4046 static void scx_ops_disable_task(struct task_struct *p)
4047 {
4048 lockdep_assert_rq_held(task_rq(p));
4049 WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED);
4050
4051 if (SCX_HAS_OP(disable))
4052 SCX_CALL_OP_TASK(SCX_KF_REST, disable, p);
4053 scx_set_task_state(p, SCX_TASK_READY);
4054 }
4055
scx_ops_exit_task(struct task_struct * p)4056 static void scx_ops_exit_task(struct task_struct *p)
4057 {
4058 struct scx_exit_task_args args = {
4059 .cancelled = false,
4060 };
4061
4062 lockdep_assert_rq_held(task_rq(p));
4063
4064 switch (scx_get_task_state(p)) {
4065 case SCX_TASK_NONE:
4066 return;
4067 case SCX_TASK_INIT:
4068 args.cancelled = true;
4069 break;
4070 case SCX_TASK_READY:
4071 break;
4072 case SCX_TASK_ENABLED:
4073 scx_ops_disable_task(p);
4074 break;
4075 default:
4076 WARN_ON_ONCE(true);
4077 return;
4078 }
4079
4080 if (SCX_HAS_OP(exit_task))
4081 SCX_CALL_OP_TASK(SCX_KF_REST, exit_task, p, &args);
4082 scx_set_task_state(p, SCX_TASK_NONE);
4083 }
4084
init_scx_entity(struct sched_ext_entity * scx)4085 void init_scx_entity(struct sched_ext_entity *scx)
4086 {
4087 memset(scx, 0, sizeof(*scx));
4088 INIT_LIST_HEAD(&scx->dsq_list.node);
4089 RB_CLEAR_NODE(&scx->dsq_priq);
4090 scx->sticky_cpu = -1;
4091 scx->holding_cpu = -1;
4092 INIT_LIST_HEAD(&scx->runnable_node);
4093 scx->runnable_at = jiffies;
4094 scx->ddsp_dsq_id = SCX_DSQ_INVALID;
4095 scx->slice = SCX_SLICE_DFL;
4096 }
4097
scx_pre_fork(struct task_struct * p)4098 void scx_pre_fork(struct task_struct *p)
4099 {
4100 /*
4101 * BPF scheduler enable/disable paths want to be able to iterate and
4102 * update all tasks which can become complex when racing forks. As
4103 * enable/disable are very cold paths, let's use a percpu_rwsem to
4104 * exclude forks.
4105 */
4106 percpu_down_read(&scx_fork_rwsem);
4107 }
4108
scx_fork(struct task_struct * p)4109 int scx_fork(struct task_struct *p)
4110 {
4111 percpu_rwsem_assert_held(&scx_fork_rwsem);
4112
4113 if (scx_ops_init_task_enabled)
4114 return scx_ops_init_task(p, task_group(p), true);
4115 else
4116 return 0;
4117 }
4118
scx_post_fork(struct task_struct * p)4119 void scx_post_fork(struct task_struct *p)
4120 {
4121 if (scx_ops_init_task_enabled) {
4122 scx_set_task_state(p, SCX_TASK_READY);
4123
4124 /*
4125 * Enable the task immediately if it's running on sched_ext.
4126 * Otherwise, it'll be enabled in switching_to_scx() if and
4127 * when it's ever configured to run with a SCHED_EXT policy.
4128 */
4129 if (p->sched_class == &ext_sched_class) {
4130 struct rq_flags rf;
4131 struct rq *rq;
4132
4133 rq = task_rq_lock(p, &rf);
4134 scx_ops_enable_task(p);
4135 task_rq_unlock(rq, p, &rf);
4136 }
4137 }
4138
4139 spin_lock_irq(&scx_tasks_lock);
4140 list_add_tail(&p->scx.tasks_node, &scx_tasks);
4141 spin_unlock_irq(&scx_tasks_lock);
4142
4143 percpu_up_read(&scx_fork_rwsem);
4144 }
4145
scx_cancel_fork(struct task_struct * p)4146 void scx_cancel_fork(struct task_struct *p)
4147 {
4148 if (scx_enabled()) {
4149 struct rq *rq;
4150 struct rq_flags rf;
4151
4152 rq = task_rq_lock(p, &rf);
4153 WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY);
4154 scx_ops_exit_task(p);
4155 task_rq_unlock(rq, p, &rf);
4156 }
4157
4158 percpu_up_read(&scx_fork_rwsem);
4159 }
4160
sched_ext_free(struct task_struct * p)4161 void sched_ext_free(struct task_struct *p)
4162 {
4163 unsigned long flags;
4164
4165 spin_lock_irqsave(&scx_tasks_lock, flags);
4166 list_del_init(&p->scx.tasks_node);
4167 spin_unlock_irqrestore(&scx_tasks_lock, flags);
4168
4169 /*
4170 * @p is off scx_tasks and wholly ours. scx_ops_enable()'s READY ->
4171 * ENABLED transitions can't race us. Disable ops for @p.
4172 */
4173 if (scx_get_task_state(p) != SCX_TASK_NONE) {
4174 struct rq_flags rf;
4175 struct rq *rq;
4176
4177 rq = task_rq_lock(p, &rf);
4178 scx_ops_exit_task(p);
4179 task_rq_unlock(rq, p, &rf);
4180 }
4181 }
4182
reweight_task_scx(struct rq * rq,struct task_struct * p,const struct load_weight * lw)4183 static void reweight_task_scx(struct rq *rq, struct task_struct *p,
4184 const struct load_weight *lw)
4185 {
4186 lockdep_assert_rq_held(task_rq(p));
4187
4188 p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
4189 if (SCX_HAS_OP(set_weight))
4190 SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx.weight);
4191 }
4192
prio_changed_scx(struct rq * rq,struct task_struct * p,int oldprio)4193 static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio)
4194 {
4195 }
4196
switching_to_scx(struct rq * rq,struct task_struct * p)4197 static void switching_to_scx(struct rq *rq, struct task_struct *p)
4198 {
4199 scx_ops_enable_task(p);
4200
4201 /*
4202 * set_cpus_allowed_scx() is not called while @p is associated with a
4203 * different scheduler class. Keep the BPF scheduler up-to-date.
4204 */
4205 if (SCX_HAS_OP(set_cpumask))
4206 SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p,
4207 (struct cpumask *)p->cpus_ptr);
4208 }
4209
switched_from_scx(struct rq * rq,struct task_struct * p)4210 static void switched_from_scx(struct rq *rq, struct task_struct *p)
4211 {
4212 scx_ops_disable_task(p);
4213 }
4214
wakeup_preempt_scx(struct rq * rq,struct task_struct * p,int wake_flags)4215 static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p,int wake_flags) {}
switched_to_scx(struct rq * rq,struct task_struct * p)4216 static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
4217
scx_check_setscheduler(struct task_struct * p,int policy)4218 int scx_check_setscheduler(struct task_struct *p, int policy)
4219 {
4220 lockdep_assert_rq_held(task_rq(p));
4221
4222 /* if disallow, reject transitioning into SCX */
4223 if (scx_enabled() && READ_ONCE(p->scx.disallow) &&
4224 p->policy != policy && policy == SCHED_EXT)
4225 return -EACCES;
4226
4227 return 0;
4228 }
4229
4230 #ifdef CONFIG_NO_HZ_FULL
scx_can_stop_tick(struct rq * rq)4231 bool scx_can_stop_tick(struct rq *rq)
4232 {
4233 struct task_struct *p = rq->curr;
4234
4235 if (scx_rq_bypassing(rq))
4236 return false;
4237
4238 if (p->sched_class != &ext_sched_class)
4239 return true;
4240
4241 /*
4242 * @rq can dispatch from different DSQs, so we can't tell whether it
4243 * needs the tick or not by looking at nr_running. Allow stopping ticks
4244 * iff the BPF scheduler indicated so. See set_next_task_scx().
4245 */
4246 return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;
4247 }
4248 #endif
4249
4250 #ifdef CONFIG_EXT_GROUP_SCHED
4251
4252 DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_rwsem);
4253 static bool scx_cgroup_enabled;
4254 static bool cgroup_warned_missing_weight;
4255 static bool cgroup_warned_missing_idle;
4256
scx_cgroup_warn_missing_weight(struct task_group * tg)4257 static void scx_cgroup_warn_missing_weight(struct task_group *tg)
4258 {
4259 if (scx_ops_enable_state() == SCX_OPS_DISABLED ||
4260 cgroup_warned_missing_weight)
4261 return;
4262
4263 if ((scx_ops.flags & SCX_OPS_HAS_CGROUP_WEIGHT) || !tg->css.parent)
4264 return;
4265
4266 pr_warn("sched_ext: \"%s\" does not implement cgroup cpu.weight\n",
4267 scx_ops.name);
4268 cgroup_warned_missing_weight = true;
4269 }
4270
scx_cgroup_warn_missing_idle(struct task_group * tg)4271 static void scx_cgroup_warn_missing_idle(struct task_group *tg)
4272 {
4273 if (!scx_cgroup_enabled || cgroup_warned_missing_idle)
4274 return;
4275
4276 if (!tg->idle)
4277 return;
4278
4279 pr_warn("sched_ext: \"%s\" does not implement cgroup cpu.idle\n",
4280 scx_ops.name);
4281 cgroup_warned_missing_idle = true;
4282 }
4283
scx_tg_online(struct task_group * tg)4284 int scx_tg_online(struct task_group *tg)
4285 {
4286 int ret = 0;
4287
4288 WARN_ON_ONCE(tg->scx_flags & (SCX_TG_ONLINE | SCX_TG_INITED));
4289
4290 percpu_down_read(&scx_cgroup_rwsem);
4291
4292 scx_cgroup_warn_missing_weight(tg);
4293
4294 if (scx_cgroup_enabled) {
4295 if (SCX_HAS_OP(cgroup_init)) {
4296 struct scx_cgroup_init_args args =
4297 { .weight = tg->scx_weight };
4298
4299 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init,
4300 tg->css.cgroup, &args);
4301 if (ret)
4302 ret = ops_sanitize_err("cgroup_init", ret);
4303 }
4304 if (ret == 0)
4305 tg->scx_flags |= SCX_TG_ONLINE | SCX_TG_INITED;
4306 } else {
4307 tg->scx_flags |= SCX_TG_ONLINE;
4308 }
4309
4310 percpu_up_read(&scx_cgroup_rwsem);
4311 return ret;
4312 }
4313
scx_tg_offline(struct task_group * tg)4314 void scx_tg_offline(struct task_group *tg)
4315 {
4316 WARN_ON_ONCE(!(tg->scx_flags & SCX_TG_ONLINE));
4317
4318 percpu_down_read(&scx_cgroup_rwsem);
4319
4320 if (SCX_HAS_OP(cgroup_exit) && (tg->scx_flags & SCX_TG_INITED))
4321 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_exit, tg->css.cgroup);
4322 tg->scx_flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED);
4323
4324 percpu_up_read(&scx_cgroup_rwsem);
4325 }
4326
scx_cgroup_can_attach(struct cgroup_taskset * tset)4327 int scx_cgroup_can_attach(struct cgroup_taskset *tset)
4328 {
4329 struct cgroup_subsys_state *css;
4330 struct task_struct *p;
4331 int ret;
4332
4333 /* released in scx_finish/cancel_attach() */
4334 percpu_down_read(&scx_cgroup_rwsem);
4335
4336 if (!scx_cgroup_enabled)
4337 return 0;
4338
4339 cgroup_taskset_for_each(p, css, tset) {
4340 struct cgroup *from = tg_cgrp(task_group(p));
4341 struct cgroup *to = tg_cgrp(css_tg(css));
4342
4343 WARN_ON_ONCE(p->scx.cgrp_moving_from);
4344
4345 /*
4346 * sched_move_task() omits identity migrations. Let's match the
4347 * behavior so that ops.cgroup_prep_move() and ops.cgroup_move()
4348 * always match one-to-one.
4349 */
4350 if (from == to)
4351 continue;
4352
4353 if (SCX_HAS_OP(cgroup_prep_move)) {
4354 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_prep_move,
4355 p, from, css->cgroup);
4356 if (ret)
4357 goto err;
4358 }
4359
4360 p->scx.cgrp_moving_from = from;
4361 }
4362
4363 return 0;
4364
4365 err:
4366 cgroup_taskset_for_each(p, css, tset) {
4367 if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from)
4368 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_cancel_move, p,
4369 p->scx.cgrp_moving_from, css->cgroup);
4370 p->scx.cgrp_moving_from = NULL;
4371 }
4372
4373 percpu_up_read(&scx_cgroup_rwsem);
4374 return ops_sanitize_err("cgroup_prep_move", ret);
4375 }
4376
scx_cgroup_move_task(struct task_struct * p)4377 void scx_cgroup_move_task(struct task_struct *p)
4378 {
4379 if (!scx_cgroup_enabled)
4380 return;
4381
4382 /*
4383 * @p must have ops.cgroup_prep_move() called on it and thus
4384 * cgrp_moving_from set.
4385 */
4386 if (SCX_HAS_OP(cgroup_move) && !WARN_ON_ONCE(!p->scx.cgrp_moving_from))
4387 SCX_CALL_OP_TASK(SCX_KF_UNLOCKED, cgroup_move, p,
4388 p->scx.cgrp_moving_from, tg_cgrp(task_group(p)));
4389 p->scx.cgrp_moving_from = NULL;
4390 }
4391
scx_cgroup_finish_attach(void)4392 void scx_cgroup_finish_attach(void)
4393 {
4394 percpu_up_read(&scx_cgroup_rwsem);
4395 }
4396
scx_cgroup_cancel_attach(struct cgroup_taskset * tset)4397 void scx_cgroup_cancel_attach(struct cgroup_taskset *tset)
4398 {
4399 struct cgroup_subsys_state *css;
4400 struct task_struct *p;
4401
4402 if (!scx_cgroup_enabled)
4403 goto out_unlock;
4404
4405 cgroup_taskset_for_each(p, css, tset) {
4406 if (SCX_HAS_OP(cgroup_cancel_move) && p->scx.cgrp_moving_from)
4407 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_cancel_move, p,
4408 p->scx.cgrp_moving_from, css->cgroup);
4409 p->scx.cgrp_moving_from = NULL;
4410 }
4411 out_unlock:
4412 percpu_up_read(&scx_cgroup_rwsem);
4413 }
4414
scx_group_set_weight(struct task_group * tg,unsigned long weight)4415 void scx_group_set_weight(struct task_group *tg, unsigned long weight)
4416 {
4417 percpu_down_read(&scx_cgroup_rwsem);
4418
4419 if (scx_cgroup_enabled && tg->scx_weight != weight) {
4420 if (SCX_HAS_OP(cgroup_set_weight))
4421 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_set_weight,
4422 tg_cgrp(tg), weight);
4423 tg->scx_weight = weight;
4424 }
4425
4426 percpu_up_read(&scx_cgroup_rwsem);
4427 }
4428
scx_group_set_idle(struct task_group * tg,bool idle)4429 void scx_group_set_idle(struct task_group *tg, bool idle)
4430 {
4431 percpu_down_read(&scx_cgroup_rwsem);
4432 scx_cgroup_warn_missing_idle(tg);
4433 percpu_up_read(&scx_cgroup_rwsem);
4434 }
4435
scx_cgroup_lock(void)4436 static void scx_cgroup_lock(void)
4437 {
4438 percpu_down_write(&scx_cgroup_rwsem);
4439 }
4440
scx_cgroup_unlock(void)4441 static void scx_cgroup_unlock(void)
4442 {
4443 percpu_up_write(&scx_cgroup_rwsem);
4444 }
4445
4446 #else /* CONFIG_EXT_GROUP_SCHED */
4447
scx_cgroup_lock(void)4448 static inline void scx_cgroup_lock(void) {}
scx_cgroup_unlock(void)4449 static inline void scx_cgroup_unlock(void) {}
4450
4451 #endif /* CONFIG_EXT_GROUP_SCHED */
4452
4453 /*
4454 * Omitted operations:
4455 *
4456 * - wakeup_preempt: NOOP as it isn't useful in the wakeup path because the task
4457 * isn't tied to the CPU at that point. Preemption is implemented by resetting
4458 * the victim task's slice to 0 and triggering reschedule on the target CPU.
4459 *
4460 * - migrate_task_rq: Unnecessary as task to cpu mapping is transient.
4461 *
4462 * - task_fork/dead: We need fork/dead notifications for all tasks regardless of
4463 * their current sched_class. Call them directly from sched core instead.
4464 */
4465 DEFINE_SCHED_CLASS(ext) = {
4466 .enqueue_task = enqueue_task_scx,
4467 .dequeue_task = dequeue_task_scx,
4468 .yield_task = yield_task_scx,
4469 .yield_to_task = yield_to_task_scx,
4470
4471 .wakeup_preempt = wakeup_preempt_scx,
4472
4473 .balance = balance_scx,
4474 .pick_task = pick_task_scx,
4475
4476 .put_prev_task = put_prev_task_scx,
4477 .set_next_task = set_next_task_scx,
4478
4479 #ifdef CONFIG_SMP
4480 .select_task_rq = select_task_rq_scx,
4481 .task_woken = task_woken_scx,
4482 .set_cpus_allowed = set_cpus_allowed_scx,
4483
4484 .rq_online = rq_online_scx,
4485 .rq_offline = rq_offline_scx,
4486 #endif
4487
4488 .task_tick = task_tick_scx,
4489
4490 .switching_to = switching_to_scx,
4491 .switched_from = switched_from_scx,
4492 .switched_to = switched_to_scx,
4493 .reweight_task = reweight_task_scx,
4494 .prio_changed = prio_changed_scx,
4495
4496 .update_curr = update_curr_scx,
4497
4498 #ifdef CONFIG_UCLAMP_TASK
4499 .uclamp_enabled = 1,
4500 #endif
4501 };
4502
init_dsq(struct scx_dispatch_q * dsq,u64 dsq_id)4503 static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id)
4504 {
4505 memset(dsq, 0, sizeof(*dsq));
4506
4507 raw_spin_lock_init(&dsq->lock);
4508 INIT_LIST_HEAD(&dsq->list);
4509 dsq->id = dsq_id;
4510 }
4511
create_dsq(u64 dsq_id,int node)4512 static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node)
4513 {
4514 struct scx_dispatch_q *dsq;
4515 int ret;
4516
4517 if (dsq_id & SCX_DSQ_FLAG_BUILTIN)
4518 return ERR_PTR(-EINVAL);
4519
4520 dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node);
4521 if (!dsq)
4522 return ERR_PTR(-ENOMEM);
4523
4524 init_dsq(dsq, dsq_id);
4525
4526 ret = rhashtable_insert_fast(&dsq_hash, &dsq->hash_node,
4527 dsq_hash_params);
4528 if (ret) {
4529 kfree(dsq);
4530 return ERR_PTR(ret);
4531 }
4532 return dsq;
4533 }
4534
free_dsq_irq_workfn(struct irq_work * irq_work)4535 static void free_dsq_irq_workfn(struct irq_work *irq_work)
4536 {
4537 struct llist_node *to_free = llist_del_all(&dsqs_to_free);
4538 struct scx_dispatch_q *dsq, *tmp_dsq;
4539
4540 llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node)
4541 kfree_rcu(dsq, rcu);
4542 }
4543
4544 static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
4545
destroy_dsq(u64 dsq_id)4546 static void destroy_dsq(u64 dsq_id)
4547 {
4548 struct scx_dispatch_q *dsq;
4549 unsigned long flags;
4550
4551 rcu_read_lock();
4552
4553 dsq = find_user_dsq(dsq_id);
4554 if (!dsq)
4555 goto out_unlock_rcu;
4556
4557 raw_spin_lock_irqsave(&dsq->lock, flags);
4558
4559 if (dsq->nr) {
4560 scx_ops_error("attempting to destroy in-use dsq 0x%016llx (nr=%u)",
4561 dsq->id, dsq->nr);
4562 goto out_unlock_dsq;
4563 }
4564
4565 if (rhashtable_remove_fast(&dsq_hash, &dsq->hash_node, dsq_hash_params))
4566 goto out_unlock_dsq;
4567
4568 /*
4569 * Mark dead by invalidating ->id to prevent dispatch_enqueue() from
4570 * queueing more tasks. As this function can be called from anywhere,
4571 * freeing is bounced through an irq work to avoid nesting RCU
4572 * operations inside scheduler locks.
4573 */
4574 dsq->id = SCX_DSQ_INVALID;
4575 llist_add(&dsq->free_node, &dsqs_to_free);
4576 irq_work_queue(&free_dsq_irq_work);
4577
4578 out_unlock_dsq:
4579 raw_spin_unlock_irqrestore(&dsq->lock, flags);
4580 out_unlock_rcu:
4581 rcu_read_unlock();
4582 }
4583
4584 #ifdef CONFIG_EXT_GROUP_SCHED
scx_cgroup_exit(void)4585 static void scx_cgroup_exit(void)
4586 {
4587 struct cgroup_subsys_state *css;
4588
4589 percpu_rwsem_assert_held(&scx_cgroup_rwsem);
4590
4591 scx_cgroup_enabled = false;
4592
4593 /*
4594 * scx_tg_on/offline() are excluded through scx_cgroup_rwsem. If we walk
4595 * cgroups and exit all the inited ones, all online cgroups are exited.
4596 */
4597 rcu_read_lock();
4598 css_for_each_descendant_post(css, &root_task_group.css) {
4599 struct task_group *tg = css_tg(css);
4600
4601 if (!(tg->scx_flags & SCX_TG_INITED))
4602 continue;
4603 tg->scx_flags &= ~SCX_TG_INITED;
4604
4605 if (!scx_ops.cgroup_exit)
4606 continue;
4607
4608 if (WARN_ON_ONCE(!css_tryget(css)))
4609 continue;
4610 rcu_read_unlock();
4611
4612 SCX_CALL_OP(SCX_KF_UNLOCKED, cgroup_exit, css->cgroup);
4613
4614 rcu_read_lock();
4615 css_put(css);
4616 }
4617 rcu_read_unlock();
4618 }
4619
scx_cgroup_init(void)4620 static int scx_cgroup_init(void)
4621 {
4622 struct cgroup_subsys_state *css;
4623 int ret;
4624
4625 percpu_rwsem_assert_held(&scx_cgroup_rwsem);
4626
4627 cgroup_warned_missing_weight = false;
4628 cgroup_warned_missing_idle = false;
4629
4630 /*
4631 * scx_tg_on/offline() are excluded through scx_cgroup_rwsem. If we walk
4632 * cgroups and init, all online cgroups are initialized.
4633 */
4634 rcu_read_lock();
4635 css_for_each_descendant_pre(css, &root_task_group.css) {
4636 struct task_group *tg = css_tg(css);
4637 struct scx_cgroup_init_args args = { .weight = tg->scx_weight };
4638
4639 scx_cgroup_warn_missing_weight(tg);
4640 scx_cgroup_warn_missing_idle(tg);
4641
4642 if ((tg->scx_flags &
4643 (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE)
4644 continue;
4645
4646 if (!scx_ops.cgroup_init) {
4647 tg->scx_flags |= SCX_TG_INITED;
4648 continue;
4649 }
4650
4651 if (WARN_ON_ONCE(!css_tryget(css)))
4652 continue;
4653 rcu_read_unlock();
4654
4655 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, cgroup_init,
4656 css->cgroup, &args);
4657 if (ret) {
4658 css_put(css);
4659 scx_ops_error("ops.cgroup_init() failed (%d)", ret);
4660 return ret;
4661 }
4662 tg->scx_flags |= SCX_TG_INITED;
4663
4664 rcu_read_lock();
4665 css_put(css);
4666 }
4667 rcu_read_unlock();
4668
4669 WARN_ON_ONCE(scx_cgroup_enabled);
4670 scx_cgroup_enabled = true;
4671
4672 return 0;
4673 }
4674
4675 #else
scx_cgroup_exit(void)4676 static void scx_cgroup_exit(void) {}
scx_cgroup_init(void)4677 static int scx_cgroup_init(void) { return 0; }
4678 #endif
4679
4680
4681 /********************************************************************************
4682 * Sysfs interface and ops enable/disable.
4683 */
4684
4685 #define SCX_ATTR(_name) \
4686 static struct kobj_attribute scx_attr_##_name = { \
4687 .attr = { .name = __stringify(_name), .mode = 0444 }, \
4688 .show = scx_attr_##_name##_show, \
4689 }
4690
scx_attr_state_show(struct kobject * kobj,struct kobj_attribute * ka,char * buf)4691 static ssize_t scx_attr_state_show(struct kobject *kobj,
4692 struct kobj_attribute *ka, char *buf)
4693 {
4694 return sysfs_emit(buf, "%s\n",
4695 scx_ops_enable_state_str[scx_ops_enable_state()]);
4696 }
4697 SCX_ATTR(state);
4698
scx_attr_switch_all_show(struct kobject * kobj,struct kobj_attribute * ka,char * buf)4699 static ssize_t scx_attr_switch_all_show(struct kobject *kobj,
4700 struct kobj_attribute *ka, char *buf)
4701 {
4702 return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all));
4703 }
4704 SCX_ATTR(switch_all);
4705
scx_attr_nr_rejected_show(struct kobject * kobj,struct kobj_attribute * ka,char * buf)4706 static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj,
4707 struct kobj_attribute *ka, char *buf)
4708 {
4709 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected));
4710 }
4711 SCX_ATTR(nr_rejected);
4712
scx_attr_hotplug_seq_show(struct kobject * kobj,struct kobj_attribute * ka,char * buf)4713 static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj,
4714 struct kobj_attribute *ka, char *buf)
4715 {
4716 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq));
4717 }
4718 SCX_ATTR(hotplug_seq);
4719
scx_attr_enable_seq_show(struct kobject * kobj,struct kobj_attribute * ka,char * buf)4720 static ssize_t scx_attr_enable_seq_show(struct kobject *kobj,
4721 struct kobj_attribute *ka, char *buf)
4722 {
4723 return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq));
4724 }
4725 SCX_ATTR(enable_seq);
4726
4727 static struct attribute *scx_global_attrs[] = {
4728 &scx_attr_state.attr,
4729 &scx_attr_switch_all.attr,
4730 &scx_attr_nr_rejected.attr,
4731 &scx_attr_hotplug_seq.attr,
4732 &scx_attr_enable_seq.attr,
4733 NULL,
4734 };
4735
4736 static const struct attribute_group scx_global_attr_group = {
4737 .attrs = scx_global_attrs,
4738 };
4739
scx_kobj_release(struct kobject * kobj)4740 static void scx_kobj_release(struct kobject *kobj)
4741 {
4742 kfree(kobj);
4743 }
4744
scx_attr_ops_show(struct kobject * kobj,struct kobj_attribute * ka,char * buf)4745 static ssize_t scx_attr_ops_show(struct kobject *kobj,
4746 struct kobj_attribute *ka, char *buf)
4747 {
4748 return sysfs_emit(buf, "%s\n", scx_ops.name);
4749 }
4750 SCX_ATTR(ops);
4751
4752 static struct attribute *scx_sched_attrs[] = {
4753 &scx_attr_ops.attr,
4754 NULL,
4755 };
4756 ATTRIBUTE_GROUPS(scx_sched);
4757
4758 static const struct kobj_type scx_ktype = {
4759 .release = scx_kobj_release,
4760 .sysfs_ops = &kobj_sysfs_ops,
4761 .default_groups = scx_sched_groups,
4762 };
4763
scx_uevent(const struct kobject * kobj,struct kobj_uevent_env * env)4764 static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)
4765 {
4766 return add_uevent_var(env, "SCXOPS=%s", scx_ops.name);
4767 }
4768
4769 static const struct kset_uevent_ops scx_uevent_ops = {
4770 .uevent = scx_uevent,
4771 };
4772
4773 /*
4774 * Used by sched_fork() and __setscheduler_prio() to pick the matching
4775 * sched_class. dl/rt are already handled.
4776 */
task_should_scx(int policy)4777 bool task_should_scx(int policy)
4778 {
4779 if (!scx_enabled() ||
4780 unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING))
4781 return false;
4782 if (READ_ONCE(scx_switching_all))
4783 return true;
4784 return policy == SCHED_EXT;
4785 }
4786
4787 /**
4788 * scx_softlockup - sched_ext softlockup handler
4789 * @dur_s: number of seconds of CPU stuck due to soft lockup
4790 *
4791 * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can
4792 * live-lock the system by making many CPUs target the same DSQ to the point
4793 * where soft-lockup detection triggers. This function is called from
4794 * soft-lockup watchdog when the triggering point is close and tries to unjam
4795 * the system by enabling the breather and aborting the BPF scheduler.
4796 */
scx_softlockup(u32 dur_s)4797 void scx_softlockup(u32 dur_s)
4798 {
4799 switch (scx_ops_enable_state()) {
4800 case SCX_OPS_ENABLING:
4801 case SCX_OPS_ENABLED:
4802 break;
4803 default:
4804 return;
4805 }
4806
4807 /* allow only one instance, cleared at the end of scx_ops_bypass() */
4808 if (test_and_set_bit(0, &scx_in_softlockup))
4809 return;
4810
4811 printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU%d stuck for %us, disabling \"%s\"\n",
4812 smp_processor_id(), dur_s, scx_ops.name);
4813
4814 /*
4815 * Some CPUs may be trapped in the dispatch paths. Enable breather
4816 * immediately; otherwise, we might even be able to get to
4817 * scx_ops_bypass().
4818 */
4819 atomic_inc(&scx_ops_breather_depth);
4820
4821 scx_ops_error("soft lockup - CPU#%d stuck for %us",
4822 smp_processor_id(), dur_s);
4823 }
4824
scx_clear_softlockup(void)4825 static void scx_clear_softlockup(void)
4826 {
4827 if (test_and_clear_bit(0, &scx_in_softlockup))
4828 atomic_dec(&scx_ops_breather_depth);
4829 }
4830
4831 /**
4832 * scx_ops_bypass - [Un]bypass scx_ops and guarantee forward progress
4833 * @bypass: true for bypass, false for unbypass
4834 *
4835 * Bypassing guarantees that all runnable tasks make forward progress without
4836 * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might
4837 * be held by tasks that the BPF scheduler is forgetting to run, which
4838 * unfortunately also excludes toggling the static branches.
4839 *
4840 * Let's work around by overriding a couple ops and modifying behaviors based on
4841 * the DISABLING state and then cycling the queued tasks through dequeue/enqueue
4842 * to force global FIFO scheduling.
4843 *
4844 * - ops.select_cpu() is ignored and the default select_cpu() is used.
4845 *
4846 * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order.
4847 * %SCX_OPS_ENQ_LAST is also ignored.
4848 *
4849 * - ops.dispatch() is ignored.
4850 *
4851 * - balance_scx() does not set %SCX_RQ_BAL_KEEP on non-zero slice as slice
4852 * can't be trusted. Whenever a tick triggers, the running task is rotated to
4853 * the tail of the queue with core_sched_at touched.
4854 *
4855 * - pick_next_task() suppresses zero slice warning.
4856 *
4857 * - scx_bpf_kick_cpu() is disabled to avoid irq_work malfunction during PM
4858 * operations.
4859 *
4860 * - scx_prio_less() reverts to the default core_sched_at order.
4861 */
scx_ops_bypass(bool bypass)4862 static void scx_ops_bypass(bool bypass)
4863 {
4864 static DEFINE_RAW_SPINLOCK(bypass_lock);
4865 int cpu;
4866 unsigned long flags;
4867
4868 raw_spin_lock_irqsave(&bypass_lock, flags);
4869 if (bypass) {
4870 scx_ops_bypass_depth++;
4871 WARN_ON_ONCE(scx_ops_bypass_depth <= 0);
4872 if (scx_ops_bypass_depth != 1)
4873 goto unlock;
4874 } else {
4875 scx_ops_bypass_depth--;
4876 WARN_ON_ONCE(scx_ops_bypass_depth < 0);
4877 if (scx_ops_bypass_depth != 0)
4878 goto unlock;
4879 }
4880
4881 atomic_inc(&scx_ops_breather_depth);
4882
4883 /*
4884 * No task property is changing. We just need to make sure all currently
4885 * queued tasks are re-queued according to the new scx_rq_bypassing()
4886 * state. As an optimization, walk each rq's runnable_list instead of
4887 * the scx_tasks list.
4888 *
4889 * This function can't trust the scheduler and thus can't use
4890 * cpus_read_lock(). Walk all possible CPUs instead of online.
4891 */
4892 for_each_possible_cpu(cpu) {
4893 struct rq *rq = cpu_rq(cpu);
4894 struct task_struct *p, *n;
4895
4896 raw_spin_rq_lock(rq);
4897
4898 if (bypass) {
4899 WARN_ON_ONCE(rq->scx.flags & SCX_RQ_BYPASSING);
4900 rq->scx.flags |= SCX_RQ_BYPASSING;
4901 } else {
4902 WARN_ON_ONCE(!(rq->scx.flags & SCX_RQ_BYPASSING));
4903 rq->scx.flags &= ~SCX_RQ_BYPASSING;
4904 }
4905
4906 /*
4907 * We need to guarantee that no tasks are on the BPF scheduler
4908 * while bypassing. Either we see enabled or the enable path
4909 * sees scx_rq_bypassing() before moving tasks to SCX.
4910 */
4911 if (!scx_enabled()) {
4912 raw_spin_rq_unlock(rq);
4913 continue;
4914 }
4915
4916 /*
4917 * The use of list_for_each_entry_safe_reverse() is required
4918 * because each task is going to be removed from and added back
4919 * to the runnable_list during iteration. Because they're added
4920 * to the tail of the list, safe reverse iteration can still
4921 * visit all nodes.
4922 */
4923 list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list,
4924 scx.runnable_node) {
4925 struct sched_enq_and_set_ctx ctx;
4926
4927 /* cycling deq/enq is enough, see the function comment */
4928 sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx);
4929 sched_enq_and_set_task(&ctx);
4930 }
4931
4932 /* resched to restore ticks and idle state */
4933 if (cpu_online(cpu) || cpu == smp_processor_id())
4934 resched_curr(rq);
4935
4936 raw_spin_rq_unlock(rq);
4937 }
4938
4939 atomic_dec(&scx_ops_breather_depth);
4940 unlock:
4941 raw_spin_unlock_irqrestore(&bypass_lock, flags);
4942 scx_clear_softlockup();
4943 }
4944
free_exit_info(struct scx_exit_info * ei)4945 static void free_exit_info(struct scx_exit_info *ei)
4946 {
4947 kfree(ei->dump);
4948 kfree(ei->msg);
4949 kfree(ei->bt);
4950 kfree(ei);
4951 }
4952
alloc_exit_info(size_t exit_dump_len)4953 static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len)
4954 {
4955 struct scx_exit_info *ei;
4956
4957 ei = kzalloc(sizeof(*ei), GFP_KERNEL);
4958 if (!ei)
4959 return NULL;
4960
4961 ei->bt = kcalloc(SCX_EXIT_BT_LEN, sizeof(ei->bt[0]), GFP_KERNEL);
4962 ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL);
4963 ei->dump = kzalloc(exit_dump_len, GFP_KERNEL);
4964
4965 if (!ei->bt || !ei->msg || !ei->dump) {
4966 free_exit_info(ei);
4967 return NULL;
4968 }
4969
4970 return ei;
4971 }
4972
scx_exit_reason(enum scx_exit_kind kind)4973 static const char *scx_exit_reason(enum scx_exit_kind kind)
4974 {
4975 switch (kind) {
4976 case SCX_EXIT_UNREG:
4977 return "unregistered from user space";
4978 case SCX_EXIT_UNREG_BPF:
4979 return "unregistered from BPF";
4980 case SCX_EXIT_UNREG_KERN:
4981 return "unregistered from the main kernel";
4982 case SCX_EXIT_SYSRQ:
4983 return "disabled by sysrq-S";
4984 case SCX_EXIT_ERROR:
4985 return "runtime error";
4986 case SCX_EXIT_ERROR_BPF:
4987 return "scx_bpf_error";
4988 case SCX_EXIT_ERROR_STALL:
4989 return "runnable task stall";
4990 default:
4991 return "<UNKNOWN>";
4992 }
4993 }
4994
scx_ops_disable_workfn(struct kthread_work * work)4995 static void scx_ops_disable_workfn(struct kthread_work *work)
4996 {
4997 struct scx_exit_info *ei = scx_exit_info;
4998 struct scx_task_iter sti;
4999 struct task_struct *p;
5000 struct rhashtable_iter rht_iter;
5001 struct scx_dispatch_q *dsq;
5002 int i, kind, cpu;
5003
5004 kind = atomic_read(&scx_exit_kind);
5005 while (true) {
5006 /*
5007 * NONE indicates that a new scx_ops has been registered since
5008 * disable was scheduled - don't kill the new ops. DONE
5009 * indicates that the ops has already been disabled.
5010 */
5011 if (kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)
5012 return;
5013 if (atomic_try_cmpxchg(&scx_exit_kind, &kind, SCX_EXIT_DONE))
5014 break;
5015 }
5016 ei->kind = kind;
5017 ei->reason = scx_exit_reason(ei->kind);
5018
5019 /* guarantee forward progress by bypassing scx_ops */
5020 scx_ops_bypass(true);
5021
5022 switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) {
5023 case SCX_OPS_DISABLING:
5024 WARN_ONCE(true, "sched_ext: duplicate disabling instance?");
5025 break;
5026 case SCX_OPS_DISABLED:
5027 pr_warn("sched_ext: ops error detected without ops (%s)\n",
5028 scx_exit_info->msg);
5029 WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) !=
5030 SCX_OPS_DISABLING);
5031 goto done;
5032 default:
5033 break;
5034 }
5035
5036 /*
5037 * Here, every runnable task is guaranteed to make forward progress and
5038 * we can safely use blocking synchronization constructs. Actually
5039 * disable ops.
5040 */
5041 mutex_lock(&scx_ops_enable_mutex);
5042
5043 static_branch_disable(&__scx_switched_all);
5044 WRITE_ONCE(scx_switching_all, false);
5045
5046 /*
5047 * Shut down cgroup support before tasks so that the cgroup attach path
5048 * doesn't race against scx_ops_exit_task().
5049 */
5050 scx_cgroup_lock();
5051 scx_cgroup_exit();
5052 scx_cgroup_unlock();
5053
5054 /*
5055 * The BPF scheduler is going away. All tasks including %TASK_DEAD ones
5056 * must be switched out and exited synchronously.
5057 */
5058 percpu_down_write(&scx_fork_rwsem);
5059
5060 scx_ops_init_task_enabled = false;
5061
5062 scx_task_iter_start(&sti);
5063 while ((p = scx_task_iter_next_locked(&sti))) {
5064 const struct sched_class *old_class = p->sched_class;
5065 const struct sched_class *new_class =
5066 __setscheduler_class(p->policy, p->prio);
5067 struct sched_enq_and_set_ctx ctx;
5068
5069 if (old_class != new_class && p->se.sched_delayed)
5070 dequeue_task(task_rq(p), p, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
5071
5072 sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx);
5073
5074 p->sched_class = new_class;
5075 check_class_changing(task_rq(p), p, old_class);
5076
5077 sched_enq_and_set_task(&ctx);
5078
5079 check_class_changed(task_rq(p), p, old_class, p->prio);
5080 scx_ops_exit_task(p);
5081 }
5082 scx_task_iter_stop(&sti);
5083 percpu_up_write(&scx_fork_rwsem);
5084
5085 /*
5086 * Invalidate all the rq clocks to prevent getting outdated
5087 * rq clocks from a previous scx scheduler.
5088 */
5089 for_each_possible_cpu(cpu) {
5090 struct rq *rq = cpu_rq(cpu);
5091 scx_rq_clock_invalidate(rq);
5092 }
5093
5094 /* no task is on scx, turn off all the switches and flush in-progress calls */
5095 static_branch_disable(&__scx_ops_enabled);
5096 for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
5097 static_branch_disable(&scx_has_op[i]);
5098 static_branch_disable(&scx_ops_enq_last);
5099 static_branch_disable(&scx_ops_enq_exiting);
5100 static_branch_disable(&scx_ops_enq_migration_disabled);
5101 static_branch_disable(&scx_ops_cpu_preempt);
5102 static_branch_disable(&scx_builtin_idle_enabled);
5103 synchronize_rcu();
5104
5105 if (ei->kind >= SCX_EXIT_ERROR) {
5106 pr_err("sched_ext: BPF scheduler \"%s\" disabled (%s)\n",
5107 scx_ops.name, ei->reason);
5108
5109 if (ei->msg[0] != '\0')
5110 pr_err("sched_ext: %s: %s\n", scx_ops.name, ei->msg);
5111 #ifdef CONFIG_STACKTRACE
5112 stack_trace_print(ei->bt, ei->bt_len, 2);
5113 #endif
5114 } else {
5115 pr_info("sched_ext: BPF scheduler \"%s\" disabled (%s)\n",
5116 scx_ops.name, ei->reason);
5117 }
5118
5119 if (scx_ops.exit)
5120 SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei);
5121
5122 cancel_delayed_work_sync(&scx_watchdog_work);
5123
5124 /*
5125 * Delete the kobject from the hierarchy eagerly in addition to just
5126 * dropping a reference. Otherwise, if the object is deleted
5127 * asynchronously, sysfs could observe an object of the same name still
5128 * in the hierarchy when another scheduler is loaded.
5129 */
5130 kobject_del(scx_root_kobj);
5131 kobject_put(scx_root_kobj);
5132 scx_root_kobj = NULL;
5133
5134 memset(&scx_ops, 0, sizeof(scx_ops));
5135
5136 rhashtable_walk_enter(&dsq_hash, &rht_iter);
5137 do {
5138 rhashtable_walk_start(&rht_iter);
5139
5140 while ((dsq = rhashtable_walk_next(&rht_iter)) && !IS_ERR(dsq))
5141 destroy_dsq(dsq->id);
5142
5143 rhashtable_walk_stop(&rht_iter);
5144 } while (dsq == ERR_PTR(-EAGAIN));
5145 rhashtable_walk_exit(&rht_iter);
5146
5147 free_percpu(scx_dsp_ctx);
5148 scx_dsp_ctx = NULL;
5149 scx_dsp_max_batch = 0;
5150
5151 free_exit_info(scx_exit_info);
5152 scx_exit_info = NULL;
5153
5154 mutex_unlock(&scx_ops_enable_mutex);
5155
5156 WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) !=
5157 SCX_OPS_DISABLING);
5158 done:
5159 scx_ops_bypass(false);
5160 }
5161
5162 static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn);
5163
schedule_scx_ops_disable_work(void)5164 static void schedule_scx_ops_disable_work(void)
5165 {
5166 struct kthread_worker *helper = READ_ONCE(scx_ops_helper);
5167
5168 /*
5169 * We may be called spuriously before the first bpf_sched_ext_reg(). If
5170 * scx_ops_helper isn't set up yet, there's nothing to do.
5171 */
5172 if (helper)
5173 kthread_queue_work(helper, &scx_ops_disable_work);
5174 }
5175
scx_ops_disable(enum scx_exit_kind kind)5176 static void scx_ops_disable(enum scx_exit_kind kind)
5177 {
5178 int none = SCX_EXIT_NONE;
5179
5180 if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE))
5181 kind = SCX_EXIT_ERROR;
5182
5183 atomic_try_cmpxchg(&scx_exit_kind, &none, kind);
5184
5185 schedule_scx_ops_disable_work();
5186 }
5187
dump_newline(struct seq_buf * s)5188 static void dump_newline(struct seq_buf *s)
5189 {
5190 trace_sched_ext_dump("");
5191
5192 /* @s may be zero sized and seq_buf triggers WARN if so */
5193 if (s->size)
5194 seq_buf_putc(s, '\n');
5195 }
5196
dump_line(struct seq_buf * s,const char * fmt,...)5197 static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...)
5198 {
5199 va_list args;
5200
5201 #ifdef CONFIG_TRACEPOINTS
5202 if (trace_sched_ext_dump_enabled()) {
5203 /* protected by scx_dump_state()::dump_lock */
5204 static char line_buf[SCX_EXIT_MSG_LEN];
5205
5206 va_start(args, fmt);
5207 vscnprintf(line_buf, sizeof(line_buf), fmt, args);
5208 va_end(args);
5209
5210 trace_sched_ext_dump(line_buf);
5211 }
5212 #endif
5213 /* @s may be zero sized and seq_buf triggers WARN if so */
5214 if (s->size) {
5215 va_start(args, fmt);
5216 seq_buf_vprintf(s, fmt, args);
5217 va_end(args);
5218
5219 seq_buf_putc(s, '\n');
5220 }
5221 }
5222
dump_stack_trace(struct seq_buf * s,const char * prefix,const unsigned long * bt,unsigned int len)5223 static void dump_stack_trace(struct seq_buf *s, const char *prefix,
5224 const unsigned long *bt, unsigned int len)
5225 {
5226 unsigned int i;
5227
5228 for (i = 0; i < len; i++)
5229 dump_line(s, "%s%pS", prefix, (void *)bt[i]);
5230 }
5231
ops_dump_init(struct seq_buf * s,const char * prefix)5232 static void ops_dump_init(struct seq_buf *s, const char *prefix)
5233 {
5234 struct scx_dump_data *dd = &scx_dump_data;
5235
5236 lockdep_assert_irqs_disabled();
5237
5238 dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */
5239 dd->first = true;
5240 dd->cursor = 0;
5241 dd->s = s;
5242 dd->prefix = prefix;
5243 }
5244
ops_dump_flush(void)5245 static void ops_dump_flush(void)
5246 {
5247 struct scx_dump_data *dd = &scx_dump_data;
5248 char *line = dd->buf.line;
5249
5250 if (!dd->cursor)
5251 return;
5252
5253 /*
5254 * There's something to flush and this is the first line. Insert a blank
5255 * line to distinguish ops dump.
5256 */
5257 if (dd->first) {
5258 dump_newline(dd->s);
5259 dd->first = false;
5260 }
5261
5262 /*
5263 * There may be multiple lines in $line. Scan and emit each line
5264 * separately.
5265 */
5266 while (true) {
5267 char *end = line;
5268 char c;
5269
5270 while (*end != '\n' && *end != '\0')
5271 end++;
5272
5273 /*
5274 * If $line overflowed, it may not have newline at the end.
5275 * Always emit with a newline.
5276 */
5277 c = *end;
5278 *end = '\0';
5279 dump_line(dd->s, "%s%s", dd->prefix, line);
5280 if (c == '\0')
5281 break;
5282
5283 /* move to the next line */
5284 end++;
5285 if (*end == '\0')
5286 break;
5287 line = end;
5288 }
5289
5290 dd->cursor = 0;
5291 }
5292
ops_dump_exit(void)5293 static void ops_dump_exit(void)
5294 {
5295 ops_dump_flush();
5296 scx_dump_data.cpu = -1;
5297 }
5298
scx_dump_task(struct seq_buf * s,struct scx_dump_ctx * dctx,struct task_struct * p,char marker)5299 static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx,
5300 struct task_struct *p, char marker)
5301 {
5302 static unsigned long bt[SCX_EXIT_BT_LEN];
5303 char dsq_id_buf[19] = "(n/a)";
5304 unsigned long ops_state = atomic_long_read(&p->scx.ops_state);
5305 unsigned int bt_len = 0;
5306
5307 if (p->scx.dsq)
5308 scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx",
5309 (unsigned long long)p->scx.dsq->id);
5310
5311 dump_newline(s);
5312 dump_line(s, " %c%c %s[%d] %+ldms",
5313 marker, task_state_to_char(p), p->comm, p->pid,
5314 jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies));
5315 dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu",
5316 scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK,
5317 p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK,
5318 ops_state >> SCX_OPSS_QSEQ_SHIFT);
5319 dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s",
5320 p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf);
5321 dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u",
5322 p->scx.dsq_vtime, p->scx.slice, p->scx.weight);
5323 dump_line(s, " cpus=%*pb", cpumask_pr_args(p->cpus_ptr));
5324
5325 if (SCX_HAS_OP(dump_task)) {
5326 ops_dump_init(s, " ");
5327 SCX_CALL_OP(SCX_KF_REST, dump_task, dctx, p);
5328 ops_dump_exit();
5329 }
5330
5331 #ifdef CONFIG_STACKTRACE
5332 bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1);
5333 #endif
5334 if (bt_len) {
5335 dump_newline(s);
5336 dump_stack_trace(s, " ", bt, bt_len);
5337 }
5338 }
5339
scx_dump_state(struct scx_exit_info * ei,size_t dump_len)5340 static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len)
5341 {
5342 static DEFINE_SPINLOCK(dump_lock);
5343 static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n";
5344 struct scx_dump_ctx dctx = {
5345 .kind = ei->kind,
5346 .exit_code = ei->exit_code,
5347 .reason = ei->reason,
5348 .at_ns = ktime_get_ns(),
5349 .at_jiffies = jiffies,
5350 };
5351 struct seq_buf s;
5352 unsigned long flags;
5353 char *buf;
5354 int cpu;
5355
5356 spin_lock_irqsave(&dump_lock, flags);
5357
5358 seq_buf_init(&s, ei->dump, dump_len);
5359
5360 if (ei->kind == SCX_EXIT_NONE) {
5361 dump_line(&s, "Debug dump triggered by %s", ei->reason);
5362 } else {
5363 dump_line(&s, "%s[%d] triggered exit kind %d:",
5364 current->comm, current->pid, ei->kind);
5365 dump_line(&s, " %s (%s)", ei->reason, ei->msg);
5366 dump_newline(&s);
5367 dump_line(&s, "Backtrace:");
5368 dump_stack_trace(&s, " ", ei->bt, ei->bt_len);
5369 }
5370
5371 if (SCX_HAS_OP(dump)) {
5372 ops_dump_init(&s, "");
5373 SCX_CALL_OP(SCX_KF_UNLOCKED, dump, &dctx);
5374 ops_dump_exit();
5375 }
5376
5377 dump_newline(&s);
5378 dump_line(&s, "CPU states");
5379 dump_line(&s, "----------");
5380
5381 for_each_possible_cpu(cpu) {
5382 struct rq *rq = cpu_rq(cpu);
5383 struct rq_flags rf;
5384 struct task_struct *p;
5385 struct seq_buf ns;
5386 size_t avail, used;
5387 bool idle;
5388
5389 rq_lock(rq, &rf);
5390
5391 idle = list_empty(&rq->scx.runnable_list) &&
5392 rq->curr->sched_class == &idle_sched_class;
5393
5394 if (idle && !SCX_HAS_OP(dump_cpu))
5395 goto next;
5396
5397 /*
5398 * We don't yet know whether ops.dump_cpu() will produce output
5399 * and we may want to skip the default CPU dump if it doesn't.
5400 * Use a nested seq_buf to generate the standard dump so that we
5401 * can decide whether to commit later.
5402 */
5403 avail = seq_buf_get_buf(&s, &buf);
5404 seq_buf_init(&ns, buf, avail);
5405
5406 dump_newline(&ns);
5407 dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu pnt_seq=%lu",
5408 cpu, rq->scx.nr_running, rq->scx.flags,
5409 rq->scx.cpu_released, rq->scx.ops_qseq,
5410 rq->scx.pnt_seq);
5411 dump_line(&ns, " curr=%s[%d] class=%ps",
5412 rq->curr->comm, rq->curr->pid,
5413 rq->curr->sched_class);
5414 if (!cpumask_empty(rq->scx.cpus_to_kick))
5415 dump_line(&ns, " cpus_to_kick : %*pb",
5416 cpumask_pr_args(rq->scx.cpus_to_kick));
5417 if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle))
5418 dump_line(&ns, " idle_to_kick : %*pb",
5419 cpumask_pr_args(rq->scx.cpus_to_kick_if_idle));
5420 if (!cpumask_empty(rq->scx.cpus_to_preempt))
5421 dump_line(&ns, " cpus_to_preempt: %*pb",
5422 cpumask_pr_args(rq->scx.cpus_to_preempt));
5423 if (!cpumask_empty(rq->scx.cpus_to_wait))
5424 dump_line(&ns, " cpus_to_wait : %*pb",
5425 cpumask_pr_args(rq->scx.cpus_to_wait));
5426
5427 used = seq_buf_used(&ns);
5428 if (SCX_HAS_OP(dump_cpu)) {
5429 ops_dump_init(&ns, " ");
5430 SCX_CALL_OP(SCX_KF_REST, dump_cpu, &dctx, cpu, idle);
5431 ops_dump_exit();
5432 }
5433
5434 /*
5435 * If idle && nothing generated by ops.dump_cpu(), there's
5436 * nothing interesting. Skip.
5437 */
5438 if (idle && used == seq_buf_used(&ns))
5439 goto next;
5440
5441 /*
5442 * $s may already have overflowed when $ns was created. If so,
5443 * calling commit on it will trigger BUG.
5444 */
5445 if (avail) {
5446 seq_buf_commit(&s, seq_buf_used(&ns));
5447 if (seq_buf_has_overflowed(&ns))
5448 seq_buf_set_overflow(&s);
5449 }
5450
5451 if (rq->curr->sched_class == &ext_sched_class)
5452 scx_dump_task(&s, &dctx, rq->curr, '*');
5453
5454 list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
5455 scx_dump_task(&s, &dctx, p, ' ');
5456 next:
5457 rq_unlock(rq, &rf);
5458 }
5459
5460 if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))
5461 memcpy(ei->dump + dump_len - sizeof(trunc_marker),
5462 trunc_marker, sizeof(trunc_marker));
5463
5464 spin_unlock_irqrestore(&dump_lock, flags);
5465 }
5466
scx_ops_error_irq_workfn(struct irq_work * irq_work)5467 static void scx_ops_error_irq_workfn(struct irq_work *irq_work)
5468 {
5469 struct scx_exit_info *ei = scx_exit_info;
5470
5471 if (ei->kind >= SCX_EXIT_ERROR)
5472 scx_dump_state(ei, scx_ops.exit_dump_len);
5473
5474 schedule_scx_ops_disable_work();
5475 }
5476
5477 static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn);
5478
scx_ops_exit_kind(enum scx_exit_kind kind,s64 exit_code,const char * fmt,...)5479 static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind,
5480 s64 exit_code,
5481 const char *fmt, ...)
5482 {
5483 struct scx_exit_info *ei = scx_exit_info;
5484 int none = SCX_EXIT_NONE;
5485 va_list args;
5486
5487 if (!atomic_try_cmpxchg(&scx_exit_kind, &none, kind))
5488 return;
5489
5490 ei->exit_code = exit_code;
5491 #ifdef CONFIG_STACKTRACE
5492 if (kind >= SCX_EXIT_ERROR)
5493 ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
5494 #endif
5495 va_start(args, fmt);
5496 vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args);
5497 va_end(args);
5498
5499 /*
5500 * Set ei->kind and ->reason for scx_dump_state(). They'll be set again
5501 * in scx_ops_disable_workfn().
5502 */
5503 ei->kind = kind;
5504 ei->reason = scx_exit_reason(ei->kind);
5505
5506 irq_work_queue(&scx_ops_error_irq_work);
5507 }
5508
scx_create_rt_helper(const char * name)5509 static struct kthread_worker *scx_create_rt_helper(const char *name)
5510 {
5511 struct kthread_worker *helper;
5512
5513 helper = kthread_run_worker(0, name);
5514 if (helper)
5515 sched_set_fifo(helper->task);
5516 return helper;
5517 }
5518
check_hotplug_seq(const struct sched_ext_ops * ops)5519 static void check_hotplug_seq(const struct sched_ext_ops *ops)
5520 {
5521 unsigned long long global_hotplug_seq;
5522
5523 /*
5524 * If a hotplug event has occurred between when a scheduler was
5525 * initialized, and when we were able to attach, exit and notify user
5526 * space about it.
5527 */
5528 if (ops->hotplug_seq) {
5529 global_hotplug_seq = atomic_long_read(&scx_hotplug_seq);
5530 if (ops->hotplug_seq != global_hotplug_seq) {
5531 scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
5532 "expected hotplug seq %llu did not match actual %llu",
5533 ops->hotplug_seq, global_hotplug_seq);
5534 }
5535 }
5536 }
5537
validate_ops(const struct sched_ext_ops * ops)5538 static int validate_ops(const struct sched_ext_ops *ops)
5539 {
5540 /*
5541 * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the
5542 * ops.enqueue() callback isn't implemented.
5543 */
5544 if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) {
5545 scx_ops_error("SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented");
5546 return -EINVAL;
5547 }
5548
5549 return 0;
5550 }
5551
scx_ops_enable(struct sched_ext_ops * ops,struct bpf_link * link)5552 static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link)
5553 {
5554 struct scx_task_iter sti;
5555 struct task_struct *p;
5556 unsigned long timeout;
5557 int i, cpu, node, ret;
5558
5559 if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN),
5560 cpu_possible_mask)) {
5561 pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n");
5562 return -EINVAL;
5563 }
5564
5565 mutex_lock(&scx_ops_enable_mutex);
5566
5567 if (!scx_ops_helper) {
5568 WRITE_ONCE(scx_ops_helper,
5569 scx_create_rt_helper("sched_ext_ops_helper"));
5570 if (!scx_ops_helper) {
5571 ret = -ENOMEM;
5572 goto err_unlock;
5573 }
5574 }
5575
5576 if (!global_dsqs) {
5577 struct scx_dispatch_q **dsqs;
5578
5579 dsqs = kcalloc(nr_node_ids, sizeof(dsqs[0]), GFP_KERNEL);
5580 if (!dsqs) {
5581 ret = -ENOMEM;
5582 goto err_unlock;
5583 }
5584
5585 for_each_node_state(node, N_POSSIBLE) {
5586 struct scx_dispatch_q *dsq;
5587
5588 dsq = kzalloc_node(sizeof(*dsq), GFP_KERNEL, node);
5589 if (!dsq) {
5590 for_each_node_state(node, N_POSSIBLE)
5591 kfree(dsqs[node]);
5592 kfree(dsqs);
5593 ret = -ENOMEM;
5594 goto err_unlock;
5595 }
5596
5597 init_dsq(dsq, SCX_DSQ_GLOBAL);
5598 dsqs[node] = dsq;
5599 }
5600
5601 global_dsqs = dsqs;
5602 }
5603
5604 if (scx_ops_enable_state() != SCX_OPS_DISABLED) {
5605 ret = -EBUSY;
5606 goto err_unlock;
5607 }
5608
5609 scx_root_kobj = kzalloc(sizeof(*scx_root_kobj), GFP_KERNEL);
5610 if (!scx_root_kobj) {
5611 ret = -ENOMEM;
5612 goto err_unlock;
5613 }
5614
5615 scx_root_kobj->kset = scx_kset;
5616 ret = kobject_init_and_add(scx_root_kobj, &scx_ktype, NULL, "root");
5617 if (ret < 0)
5618 goto err;
5619
5620 scx_exit_info = alloc_exit_info(ops->exit_dump_len);
5621 if (!scx_exit_info) {
5622 ret = -ENOMEM;
5623 goto err_del;
5624 }
5625
5626 /*
5627 * Set scx_ops, transition to ENABLING and clear exit info to arm the
5628 * disable path. Failure triggers full disabling from here on.
5629 */
5630 scx_ops = *ops;
5631
5632 WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_ENABLING) !=
5633 SCX_OPS_DISABLED);
5634
5635 atomic_set(&scx_exit_kind, SCX_EXIT_NONE);
5636 scx_warned_zero_slice = false;
5637
5638 atomic_long_set(&scx_nr_rejected, 0);
5639
5640 for_each_possible_cpu(cpu)
5641 cpu_rq(cpu)->scx.cpuperf_target = SCX_CPUPERF_ONE;
5642
5643 /*
5644 * Keep CPUs stable during enable so that the BPF scheduler can track
5645 * online CPUs by watching ->on/offline_cpu() after ->init().
5646 */
5647 cpus_read_lock();
5648
5649 if (scx_ops.init) {
5650 ret = SCX_CALL_OP_RET(SCX_KF_UNLOCKED, init);
5651 if (ret) {
5652 ret = ops_sanitize_err("init", ret);
5653 cpus_read_unlock();
5654 scx_ops_error("ops.init() failed (%d)", ret);
5655 goto err_disable;
5656 }
5657 }
5658
5659 for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)
5660 if (((void (**)(void))ops)[i])
5661 static_branch_enable_cpuslocked(&scx_has_op[i]);
5662
5663 check_hotplug_seq(ops);
5664 #ifdef CONFIG_SMP
5665 update_selcpu_topology();
5666 #endif
5667 cpus_read_unlock();
5668
5669 ret = validate_ops(ops);
5670 if (ret)
5671 goto err_disable;
5672
5673 WARN_ON_ONCE(scx_dsp_ctx);
5674 scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
5675 scx_dsp_ctx = __alloc_percpu(struct_size_t(struct scx_dsp_ctx, buf,
5676 scx_dsp_max_batch),
5677 __alignof__(struct scx_dsp_ctx));
5678 if (!scx_dsp_ctx) {
5679 ret = -ENOMEM;
5680 goto err_disable;
5681 }
5682
5683 if (ops->timeout_ms)
5684 timeout = msecs_to_jiffies(ops->timeout_ms);
5685 else
5686 timeout = SCX_WATCHDOG_MAX_TIMEOUT;
5687
5688 WRITE_ONCE(scx_watchdog_timeout, timeout);
5689 WRITE_ONCE(scx_watchdog_timestamp, jiffies);
5690 queue_delayed_work(system_unbound_wq, &scx_watchdog_work,
5691 scx_watchdog_timeout / 2);
5692
5693 /*
5694 * Once __scx_ops_enabled is set, %current can be switched to SCX
5695 * anytime. This can lead to stalls as some BPF schedulers (e.g.
5696 * userspace scheduling) may not function correctly before all tasks are
5697 * switched. Init in bypass mode to guarantee forward progress.
5698 */
5699 scx_ops_bypass(true);
5700
5701 for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++)
5702 if (((void (**)(void))ops)[i])
5703 static_branch_enable(&scx_has_op[i]);
5704
5705 if (ops->flags & SCX_OPS_ENQ_LAST)
5706 static_branch_enable(&scx_ops_enq_last);
5707
5708 if (ops->flags & SCX_OPS_ENQ_EXITING)
5709 static_branch_enable(&scx_ops_enq_exiting);
5710 if (ops->flags & SCX_OPS_ENQ_MIGRATION_DISABLED)
5711 static_branch_enable(&scx_ops_enq_migration_disabled);
5712 if (scx_ops.cpu_acquire || scx_ops.cpu_release)
5713 static_branch_enable(&scx_ops_cpu_preempt);
5714
5715 if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) {
5716 reset_idle_masks();
5717 static_branch_enable(&scx_builtin_idle_enabled);
5718 } else {
5719 static_branch_disable(&scx_builtin_idle_enabled);
5720 }
5721
5722 /*
5723 * Lock out forks, cgroup on/offlining and moves before opening the
5724 * floodgate so that they don't wander into the operations prematurely.
5725 */
5726 percpu_down_write(&scx_fork_rwsem);
5727
5728 WARN_ON_ONCE(scx_ops_init_task_enabled);
5729 scx_ops_init_task_enabled = true;
5730
5731 /*
5732 * Enable ops for every task. Fork is excluded by scx_fork_rwsem
5733 * preventing new tasks from being added. No need to exclude tasks
5734 * leaving as sched_ext_free() can handle both prepped and enabled
5735 * tasks. Prep all tasks first and then enable them with preemption
5736 * disabled.
5737 *
5738 * All cgroups should be initialized before scx_ops_init_task() so that
5739 * the BPF scheduler can reliably track each task's cgroup membership
5740 * from scx_ops_init_task(). Lock out cgroup on/offlining and task
5741 * migrations while tasks are being initialized so that
5742 * scx_cgroup_can_attach() never sees uninitialized tasks.
5743 */
5744 scx_cgroup_lock();
5745 ret = scx_cgroup_init();
5746 if (ret)
5747 goto err_disable_unlock_all;
5748
5749 scx_task_iter_start(&sti);
5750 while ((p = scx_task_iter_next_locked(&sti))) {
5751 /*
5752 * @p may already be dead, have lost all its usages counts and
5753 * be waiting for RCU grace period before being freed. @p can't
5754 * be initialized for SCX in such cases and should be ignored.
5755 */
5756 if (!tryget_task_struct(p))
5757 continue;
5758
5759 scx_task_iter_unlock(&sti);
5760
5761 ret = scx_ops_init_task(p, task_group(p), false);
5762 if (ret) {
5763 put_task_struct(p);
5764 scx_task_iter_relock(&sti);
5765 scx_task_iter_stop(&sti);
5766 scx_ops_error("ops.init_task() failed (%d) for %s[%d]",
5767 ret, p->comm, p->pid);
5768 goto err_disable_unlock_all;
5769 }
5770
5771 scx_set_task_state(p, SCX_TASK_READY);
5772
5773 put_task_struct(p);
5774 scx_task_iter_relock(&sti);
5775 }
5776 scx_task_iter_stop(&sti);
5777 scx_cgroup_unlock();
5778 percpu_up_write(&scx_fork_rwsem);
5779
5780 /*
5781 * All tasks are READY. It's safe to turn on scx_enabled() and switch
5782 * all eligible tasks.
5783 */
5784 WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL));
5785 static_branch_enable(&__scx_ops_enabled);
5786
5787 /*
5788 * We're fully committed and can't fail. The task READY -> ENABLED
5789 * transitions here are synchronized against sched_ext_free() through
5790 * scx_tasks_lock.
5791 */
5792 percpu_down_write(&scx_fork_rwsem);
5793 scx_task_iter_start(&sti);
5794 while ((p = scx_task_iter_next_locked(&sti))) {
5795 const struct sched_class *old_class = p->sched_class;
5796 const struct sched_class *new_class =
5797 __setscheduler_class(p->policy, p->prio);
5798 struct sched_enq_and_set_ctx ctx;
5799
5800 if (old_class != new_class && p->se.sched_delayed)
5801 dequeue_task(task_rq(p), p, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
5802
5803 sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx);
5804
5805 p->scx.slice = SCX_SLICE_DFL;
5806 p->sched_class = new_class;
5807 check_class_changing(task_rq(p), p, old_class);
5808
5809 sched_enq_and_set_task(&ctx);
5810
5811 check_class_changed(task_rq(p), p, old_class, p->prio);
5812 }
5813 scx_task_iter_stop(&sti);
5814 percpu_up_write(&scx_fork_rwsem);
5815
5816 scx_ops_bypass(false);
5817
5818 if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) {
5819 WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE);
5820 goto err_disable;
5821 }
5822
5823 if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
5824 static_branch_enable(&__scx_switched_all);
5825
5826 pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n",
5827 scx_ops.name, scx_switched_all() ? "" : " (partial)");
5828 kobject_uevent(scx_root_kobj, KOBJ_ADD);
5829 mutex_unlock(&scx_ops_enable_mutex);
5830
5831 atomic_long_inc(&scx_enable_seq);
5832
5833 return 0;
5834
5835 err_del:
5836 kobject_del(scx_root_kobj);
5837 err:
5838 kobject_put(scx_root_kobj);
5839 scx_root_kobj = NULL;
5840 if (scx_exit_info) {
5841 free_exit_info(scx_exit_info);
5842 scx_exit_info = NULL;
5843 }
5844 err_unlock:
5845 mutex_unlock(&scx_ops_enable_mutex);
5846 return ret;
5847
5848 err_disable_unlock_all:
5849 scx_cgroup_unlock();
5850 percpu_up_write(&scx_fork_rwsem);
5851 scx_ops_bypass(false);
5852 err_disable:
5853 mutex_unlock(&scx_ops_enable_mutex);
5854 /*
5855 * Returning an error code here would not pass all the error information
5856 * to userspace. Record errno using scx_ops_error() for cases
5857 * scx_ops_error() wasn't already invoked and exit indicating success so
5858 * that the error is notified through ops.exit() with all the details.
5859 *
5860 * Flush scx_ops_disable_work to ensure that error is reported before
5861 * init completion.
5862 */
5863 scx_ops_error("scx_ops_enable() failed (%d)", ret);
5864 kthread_flush_work(&scx_ops_disable_work);
5865 return 0;
5866 }
5867
5868
5869 /********************************************************************************
5870 * bpf_struct_ops plumbing.
5871 */
5872 #include <linux/bpf_verifier.h>
5873 #include <linux/bpf.h>
5874 #include <linux/btf.h>
5875
5876 static const struct btf_type *task_struct_type;
5877
bpf_scx_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)5878 static bool bpf_scx_is_valid_access(int off, int size,
5879 enum bpf_access_type type,
5880 const struct bpf_prog *prog,
5881 struct bpf_insn_access_aux *info)
5882 {
5883 if (type != BPF_READ)
5884 return false;
5885 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
5886 return false;
5887 if (off % size != 0)
5888 return false;
5889
5890 return btf_ctx_access(off, size, type, prog, info);
5891 }
5892
bpf_scx_btf_struct_access(struct bpf_verifier_log * log,const struct bpf_reg_state * reg,int off,int size)5893 static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
5894 const struct bpf_reg_state *reg, int off,
5895 int size)
5896 {
5897 const struct btf_type *t;
5898
5899 t = btf_type_by_id(reg->btf, reg->btf_id);
5900 if (t == task_struct_type) {
5901 if (off >= offsetof(struct task_struct, scx.slice) &&
5902 off + size <= offsetofend(struct task_struct, scx.slice))
5903 return SCALAR_VALUE;
5904 if (off >= offsetof(struct task_struct, scx.dsq_vtime) &&
5905 off + size <= offsetofend(struct task_struct, scx.dsq_vtime))
5906 return SCALAR_VALUE;
5907 if (off >= offsetof(struct task_struct, scx.disallow) &&
5908 off + size <= offsetofend(struct task_struct, scx.disallow))
5909 return SCALAR_VALUE;
5910 }
5911
5912 return -EACCES;
5913 }
5914
5915 static const struct bpf_func_proto *
bpf_scx_get_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)5916 bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
5917 {
5918 switch (func_id) {
5919 case BPF_FUNC_task_storage_get:
5920 return &bpf_task_storage_get_proto;
5921 case BPF_FUNC_task_storage_delete:
5922 return &bpf_task_storage_delete_proto;
5923 default:
5924 return bpf_base_func_proto(func_id, prog);
5925 }
5926 }
5927
5928 static const struct bpf_verifier_ops bpf_scx_verifier_ops = {
5929 .get_func_proto = bpf_scx_get_func_proto,
5930 .is_valid_access = bpf_scx_is_valid_access,
5931 .btf_struct_access = bpf_scx_btf_struct_access,
5932 };
5933
bpf_scx_init_member(const struct btf_type * t,const struct btf_member * member,void * kdata,const void * udata)5934 static int bpf_scx_init_member(const struct btf_type *t,
5935 const struct btf_member *member,
5936 void *kdata, const void *udata)
5937 {
5938 const struct sched_ext_ops *uops = udata;
5939 struct sched_ext_ops *ops = kdata;
5940 u32 moff = __btf_member_bit_offset(t, member) / 8;
5941 int ret;
5942
5943 switch (moff) {
5944 case offsetof(struct sched_ext_ops, dispatch_max_batch):
5945 if (*(u32 *)(udata + moff) > INT_MAX)
5946 return -E2BIG;
5947 ops->dispatch_max_batch = *(u32 *)(udata + moff);
5948 return 1;
5949 case offsetof(struct sched_ext_ops, flags):
5950 if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS)
5951 return -EINVAL;
5952 ops->flags = *(u64 *)(udata + moff);
5953 return 1;
5954 case offsetof(struct sched_ext_ops, name):
5955 ret = bpf_obj_name_cpy(ops->name, uops->name,
5956 sizeof(ops->name));
5957 if (ret < 0)
5958 return ret;
5959 if (ret == 0)
5960 return -EINVAL;
5961 return 1;
5962 case offsetof(struct sched_ext_ops, timeout_ms):
5963 if (msecs_to_jiffies(*(u32 *)(udata + moff)) >
5964 SCX_WATCHDOG_MAX_TIMEOUT)
5965 return -E2BIG;
5966 ops->timeout_ms = *(u32 *)(udata + moff);
5967 return 1;
5968 case offsetof(struct sched_ext_ops, exit_dump_len):
5969 ops->exit_dump_len =
5970 *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN;
5971 return 1;
5972 case offsetof(struct sched_ext_ops, hotplug_seq):
5973 ops->hotplug_seq = *(u64 *)(udata + moff);
5974 return 1;
5975 }
5976
5977 return 0;
5978 }
5979
bpf_scx_check_member(const struct btf_type * t,const struct btf_member * member,const struct bpf_prog * prog)5980 static int bpf_scx_check_member(const struct btf_type *t,
5981 const struct btf_member *member,
5982 const struct bpf_prog *prog)
5983 {
5984 u32 moff = __btf_member_bit_offset(t, member) / 8;
5985
5986 switch (moff) {
5987 case offsetof(struct sched_ext_ops, init_task):
5988 #ifdef CONFIG_EXT_GROUP_SCHED
5989 case offsetof(struct sched_ext_ops, cgroup_init):
5990 case offsetof(struct sched_ext_ops, cgroup_exit):
5991 case offsetof(struct sched_ext_ops, cgroup_prep_move):
5992 #endif
5993 case offsetof(struct sched_ext_ops, cpu_online):
5994 case offsetof(struct sched_ext_ops, cpu_offline):
5995 case offsetof(struct sched_ext_ops, init):
5996 case offsetof(struct sched_ext_ops, exit):
5997 break;
5998 default:
5999 if (prog->sleepable)
6000 return -EINVAL;
6001 }
6002
6003 return 0;
6004 }
6005
bpf_scx_reg(void * kdata,struct bpf_link * link)6006 static int bpf_scx_reg(void *kdata, struct bpf_link *link)
6007 {
6008 return scx_ops_enable(kdata, link);
6009 }
6010
bpf_scx_unreg(void * kdata,struct bpf_link * link)6011 static void bpf_scx_unreg(void *kdata, struct bpf_link *link)
6012 {
6013 scx_ops_disable(SCX_EXIT_UNREG);
6014 kthread_flush_work(&scx_ops_disable_work);
6015 }
6016
bpf_scx_init(struct btf * btf)6017 static int bpf_scx_init(struct btf *btf)
6018 {
6019 task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]);
6020
6021 return 0;
6022 }
6023
bpf_scx_update(void * kdata,void * old_kdata,struct bpf_link * link)6024 static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link)
6025 {
6026 /*
6027 * sched_ext does not support updating the actively-loaded BPF
6028 * scheduler, as registering a BPF scheduler can always fail if the
6029 * scheduler returns an error code for e.g. ops.init(), ops.init_task(),
6030 * etc. Similarly, we can always race with unregistration happening
6031 * elsewhere, such as with sysrq.
6032 */
6033 return -EOPNOTSUPP;
6034 }
6035
bpf_scx_validate(void * kdata)6036 static int bpf_scx_validate(void *kdata)
6037 {
6038 return 0;
6039 }
6040
sched_ext_ops__select_cpu(struct task_struct * p,s32 prev_cpu,u64 wake_flags)6041 static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; }
sched_ext_ops__enqueue(struct task_struct * p,u64 enq_flags)6042 static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {}
sched_ext_ops__dequeue(struct task_struct * p,u64 enq_flags)6043 static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {}
sched_ext_ops__dispatch(s32 prev_cpu,struct task_struct * prev__nullable)6044 static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {}
sched_ext_ops__tick(struct task_struct * p)6045 static void sched_ext_ops__tick(struct task_struct *p) {}
sched_ext_ops__runnable(struct task_struct * p,u64 enq_flags)6046 static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {}
sched_ext_ops__running(struct task_struct * p)6047 static void sched_ext_ops__running(struct task_struct *p) {}
sched_ext_ops__stopping(struct task_struct * p,bool runnable)6048 static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {}
sched_ext_ops__quiescent(struct task_struct * p,u64 deq_flags)6049 static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {}
sched_ext_ops__yield(struct task_struct * from,struct task_struct * to__nullable)6050 static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; }
sched_ext_ops__core_sched_before(struct task_struct * a,struct task_struct * b)6051 static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; }
sched_ext_ops__set_weight(struct task_struct * p,u32 weight)6052 static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {}
sched_ext_ops__set_cpumask(struct task_struct * p,const struct cpumask * mask)6053 static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {}
sched_ext_ops__update_idle(s32 cpu,bool idle)6054 static void sched_ext_ops__update_idle(s32 cpu, bool idle) {}
sched_ext_ops__cpu_acquire(s32 cpu,struct scx_cpu_acquire_args * args)6055 static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {}
sched_ext_ops__cpu_release(s32 cpu,struct scx_cpu_release_args * args)6056 static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {}
sched_ext_ops__init_task(struct task_struct * p,struct scx_init_task_args * args)6057 static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; }
sched_ext_ops__exit_task(struct task_struct * p,struct scx_exit_task_args * args)6058 static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {}
sched_ext_ops__enable(struct task_struct * p)6059 static void sched_ext_ops__enable(struct task_struct *p) {}
sched_ext_ops__disable(struct task_struct * p)6060 static void sched_ext_ops__disable(struct task_struct *p) {}
6061 #ifdef CONFIG_EXT_GROUP_SCHED
sched_ext_ops__cgroup_init(struct cgroup * cgrp,struct scx_cgroup_init_args * args)6062 static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; }
sched_ext_ops__cgroup_exit(struct cgroup * cgrp)6063 static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {}
sched_ext_ops__cgroup_prep_move(struct task_struct * p,struct cgroup * from,struct cgroup * to)6064 static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; }
sched_ext_ops__cgroup_move(struct task_struct * p,struct cgroup * from,struct cgroup * to)6065 static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {}
sched_ext_ops__cgroup_cancel_move(struct task_struct * p,struct cgroup * from,struct cgroup * to)6066 static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {}
sched_ext_ops__cgroup_set_weight(struct cgroup * cgrp,u32 weight)6067 static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {}
6068 #endif
sched_ext_ops__cpu_online(s32 cpu)6069 static void sched_ext_ops__cpu_online(s32 cpu) {}
sched_ext_ops__cpu_offline(s32 cpu)6070 static void sched_ext_ops__cpu_offline(s32 cpu) {}
sched_ext_ops__init(void)6071 static s32 sched_ext_ops__init(void) { return -EINVAL; }
sched_ext_ops__exit(struct scx_exit_info * info)6072 static void sched_ext_ops__exit(struct scx_exit_info *info) {}
sched_ext_ops__dump(struct scx_dump_ctx * ctx)6073 static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {}
sched_ext_ops__dump_cpu(struct scx_dump_ctx * ctx,s32 cpu,bool idle)6074 static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {}
sched_ext_ops__dump_task(struct scx_dump_ctx * ctx,struct task_struct * p)6075 static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {}
6076
6077 static struct sched_ext_ops __bpf_ops_sched_ext_ops = {
6078 .select_cpu = sched_ext_ops__select_cpu,
6079 .enqueue = sched_ext_ops__enqueue,
6080 .dequeue = sched_ext_ops__dequeue,
6081 .dispatch = sched_ext_ops__dispatch,
6082 .tick = sched_ext_ops__tick,
6083 .runnable = sched_ext_ops__runnable,
6084 .running = sched_ext_ops__running,
6085 .stopping = sched_ext_ops__stopping,
6086 .quiescent = sched_ext_ops__quiescent,
6087 .yield = sched_ext_ops__yield,
6088 .core_sched_before = sched_ext_ops__core_sched_before,
6089 .set_weight = sched_ext_ops__set_weight,
6090 .set_cpumask = sched_ext_ops__set_cpumask,
6091 .update_idle = sched_ext_ops__update_idle,
6092 .cpu_acquire = sched_ext_ops__cpu_acquire,
6093 .cpu_release = sched_ext_ops__cpu_release,
6094 .init_task = sched_ext_ops__init_task,
6095 .exit_task = sched_ext_ops__exit_task,
6096 .enable = sched_ext_ops__enable,
6097 .disable = sched_ext_ops__disable,
6098 #ifdef CONFIG_EXT_GROUP_SCHED
6099 .cgroup_init = sched_ext_ops__cgroup_init,
6100 .cgroup_exit = sched_ext_ops__cgroup_exit,
6101 .cgroup_prep_move = sched_ext_ops__cgroup_prep_move,
6102 .cgroup_move = sched_ext_ops__cgroup_move,
6103 .cgroup_cancel_move = sched_ext_ops__cgroup_cancel_move,
6104 .cgroup_set_weight = sched_ext_ops__cgroup_set_weight,
6105 #endif
6106 .cpu_online = sched_ext_ops__cpu_online,
6107 .cpu_offline = sched_ext_ops__cpu_offline,
6108 .init = sched_ext_ops__init,
6109 .exit = sched_ext_ops__exit,
6110 .dump = sched_ext_ops__dump,
6111 .dump_cpu = sched_ext_ops__dump_cpu,
6112 .dump_task = sched_ext_ops__dump_task,
6113 };
6114
6115 static struct bpf_struct_ops bpf_sched_ext_ops = {
6116 .verifier_ops = &bpf_scx_verifier_ops,
6117 .reg = bpf_scx_reg,
6118 .unreg = bpf_scx_unreg,
6119 .check_member = bpf_scx_check_member,
6120 .init_member = bpf_scx_init_member,
6121 .init = bpf_scx_init,
6122 .update = bpf_scx_update,
6123 .validate = bpf_scx_validate,
6124 .name = "sched_ext_ops",
6125 .owner = THIS_MODULE,
6126 .cfi_stubs = &__bpf_ops_sched_ext_ops
6127 };
6128
6129
6130 /********************************************************************************
6131 * System integration and init.
6132 */
6133
sysrq_handle_sched_ext_reset(u8 key)6134 static void sysrq_handle_sched_ext_reset(u8 key)
6135 {
6136 if (scx_ops_helper)
6137 scx_ops_disable(SCX_EXIT_SYSRQ);
6138 else
6139 pr_info("sched_ext: BPF scheduler not yet used\n");
6140 }
6141
6142 static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
6143 .handler = sysrq_handle_sched_ext_reset,
6144 .help_msg = "reset-sched-ext(S)",
6145 .action_msg = "Disable sched_ext and revert all tasks to CFS",
6146 .enable_mask = SYSRQ_ENABLE_RTNICE,
6147 };
6148
sysrq_handle_sched_ext_dump(u8 key)6149 static void sysrq_handle_sched_ext_dump(u8 key)
6150 {
6151 struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" };
6152
6153 if (scx_enabled())
6154 scx_dump_state(&ei, 0);
6155 }
6156
6157 static const struct sysrq_key_op sysrq_sched_ext_dump_op = {
6158 .handler = sysrq_handle_sched_ext_dump,
6159 .help_msg = "dump-sched-ext(D)",
6160 .action_msg = "Trigger sched_ext debug dump",
6161 .enable_mask = SYSRQ_ENABLE_RTNICE,
6162 };
6163
can_skip_idle_kick(struct rq * rq)6164 static bool can_skip_idle_kick(struct rq *rq)
6165 {
6166 lockdep_assert_rq_held(rq);
6167
6168 /*
6169 * We can skip idle kicking if @rq is going to go through at least one
6170 * full SCX scheduling cycle before going idle. Just checking whether
6171 * curr is not idle is insufficient because we could be racing
6172 * balance_one() trying to pull the next task from a remote rq, which
6173 * may fail, and @rq may become idle afterwards.
6174 *
6175 * The race window is small and we don't and can't guarantee that @rq is
6176 * only kicked while idle anyway. Skip only when sure.
6177 */
6178 return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE);
6179 }
6180
kick_one_cpu(s32 cpu,struct rq * this_rq,unsigned long * pseqs)6181 static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *pseqs)
6182 {
6183 struct rq *rq = cpu_rq(cpu);
6184 struct scx_rq *this_scx = &this_rq->scx;
6185 bool should_wait = false;
6186 unsigned long flags;
6187
6188 raw_spin_rq_lock_irqsave(rq, flags);
6189
6190 /*
6191 * During CPU hotplug, a CPU may depend on kicking itself to make
6192 * forward progress. Allow kicking self regardless of online state.
6193 */
6194 if (cpu_online(cpu) || cpu == cpu_of(this_rq)) {
6195 if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) {
6196 if (rq->curr->sched_class == &ext_sched_class)
6197 rq->curr->scx.slice = 0;
6198 cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt);
6199 }
6200
6201 if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) {
6202 pseqs[cpu] = rq->scx.pnt_seq;
6203 should_wait = true;
6204 }
6205
6206 resched_curr(rq);
6207 } else {
6208 cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt);
6209 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait);
6210 }
6211
6212 raw_spin_rq_unlock_irqrestore(rq, flags);
6213
6214 return should_wait;
6215 }
6216
kick_one_cpu_if_idle(s32 cpu,struct rq * this_rq)6217 static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq)
6218 {
6219 struct rq *rq = cpu_rq(cpu);
6220 unsigned long flags;
6221
6222 raw_spin_rq_lock_irqsave(rq, flags);
6223
6224 if (!can_skip_idle_kick(rq) &&
6225 (cpu_online(cpu) || cpu == cpu_of(this_rq)))
6226 resched_curr(rq);
6227
6228 raw_spin_rq_unlock_irqrestore(rq, flags);
6229 }
6230
kick_cpus_irq_workfn(struct irq_work * irq_work)6231 static void kick_cpus_irq_workfn(struct irq_work *irq_work)
6232 {
6233 struct rq *this_rq = this_rq();
6234 struct scx_rq *this_scx = &this_rq->scx;
6235 unsigned long *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs);
6236 bool should_wait = false;
6237 s32 cpu;
6238
6239 for_each_cpu(cpu, this_scx->cpus_to_kick) {
6240 should_wait |= kick_one_cpu(cpu, this_rq, pseqs);
6241 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick);
6242 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle);
6243 }
6244
6245 for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) {
6246 kick_one_cpu_if_idle(cpu, this_rq);
6247 cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle);
6248 }
6249
6250 if (!should_wait)
6251 return;
6252
6253 for_each_cpu(cpu, this_scx->cpus_to_wait) {
6254 unsigned long *wait_pnt_seq = &cpu_rq(cpu)->scx.pnt_seq;
6255
6256 if (cpu != cpu_of(this_rq)) {
6257 /*
6258 * Pairs with smp_store_release() issued by this CPU in
6259 * switch_class() on the resched path.
6260 *
6261 * We busy-wait here to guarantee that no other task can
6262 * be scheduled on our core before the target CPU has
6263 * entered the resched path.
6264 */
6265 while (smp_load_acquire(wait_pnt_seq) == pseqs[cpu])
6266 cpu_relax();
6267 }
6268
6269 cpumask_clear_cpu(cpu, this_scx->cpus_to_wait);
6270 }
6271 }
6272
6273 /**
6274 * print_scx_info - print out sched_ext scheduler state
6275 * @log_lvl: the log level to use when printing
6276 * @p: target task
6277 *
6278 * If a sched_ext scheduler is enabled, print the name and state of the
6279 * scheduler. If @p is on sched_ext, print further information about the task.
6280 *
6281 * This function can be safely called on any task as long as the task_struct
6282 * itself is accessible. While safe, this function isn't synchronized and may
6283 * print out mixups or garbages of limited length.
6284 */
print_scx_info(const char * log_lvl,struct task_struct * p)6285 void print_scx_info(const char *log_lvl, struct task_struct *p)
6286 {
6287 enum scx_ops_enable_state state = scx_ops_enable_state();
6288 const char *all = READ_ONCE(scx_switching_all) ? "+all" : "";
6289 char runnable_at_buf[22] = "?";
6290 struct sched_class *class;
6291 unsigned long runnable_at;
6292
6293 if (state == SCX_OPS_DISABLED)
6294 return;
6295
6296 /*
6297 * Carefully check if the task was running on sched_ext, and then
6298 * carefully copy the time it's been runnable, and its state.
6299 */
6300 if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) ||
6301 class != &ext_sched_class) {
6302 printk("%sSched_ext: %s (%s%s)", log_lvl, scx_ops.name,
6303 scx_ops_enable_state_str[state], all);
6304 return;
6305 }
6306
6307 if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at,
6308 sizeof(runnable_at)))
6309 scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms",
6310 jiffies_delta_msecs(runnable_at, jiffies));
6311
6312 /* print everything onto one line to conserve console space */
6313 printk("%sSched_ext: %s (%s%s), task: runnable_at=%s",
6314 log_lvl, scx_ops.name, scx_ops_enable_state_str[state], all,
6315 runnable_at_buf);
6316 }
6317
scx_pm_handler(struct notifier_block * nb,unsigned long event,void * ptr)6318 static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr)
6319 {
6320 /*
6321 * SCX schedulers often have userspace components which are sometimes
6322 * involved in critial scheduling paths. PM operations involve freezing
6323 * userspace which can lead to scheduling misbehaviors including stalls.
6324 * Let's bypass while PM operations are in progress.
6325 */
6326 switch (event) {
6327 case PM_HIBERNATION_PREPARE:
6328 case PM_SUSPEND_PREPARE:
6329 case PM_RESTORE_PREPARE:
6330 scx_ops_bypass(true);
6331 break;
6332 case PM_POST_HIBERNATION:
6333 case PM_POST_SUSPEND:
6334 case PM_POST_RESTORE:
6335 scx_ops_bypass(false);
6336 break;
6337 }
6338
6339 return NOTIFY_OK;
6340 }
6341
6342 static struct notifier_block scx_pm_notifier = {
6343 .notifier_call = scx_pm_handler,
6344 };
6345
init_sched_ext_class(void)6346 void __init init_sched_ext_class(void)
6347 {
6348 s32 cpu, v;
6349
6350 /*
6351 * The following is to prevent the compiler from optimizing out the enum
6352 * definitions so that BPF scheduler implementations can use them
6353 * through the generated vmlinux.h.
6354 */
6355 WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT |
6356 SCX_TG_ONLINE);
6357
6358 BUG_ON(rhashtable_init(&dsq_hash, &dsq_hash_params));
6359 #ifdef CONFIG_SMP
6360 BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL));
6361 BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL));
6362 #endif
6363 scx_kick_cpus_pnt_seqs =
6364 __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * nr_cpu_ids,
6365 __alignof__(scx_kick_cpus_pnt_seqs[0]));
6366 BUG_ON(!scx_kick_cpus_pnt_seqs);
6367
6368 for_each_possible_cpu(cpu) {
6369 struct rq *rq = cpu_rq(cpu);
6370
6371 init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL);
6372 INIT_LIST_HEAD(&rq->scx.runnable_list);
6373 INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
6374
6375 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick, GFP_KERNEL));
6376 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL));
6377 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_preempt, GFP_KERNEL));
6378 BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_wait, GFP_KERNEL));
6379 init_irq_work(&rq->scx.deferred_irq_work, deferred_irq_workfn);
6380 init_irq_work(&rq->scx.kick_cpus_irq_work, kick_cpus_irq_workfn);
6381
6382 if (cpu_online(cpu))
6383 cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE;
6384 }
6385
6386 register_sysrq_key('S', &sysrq_sched_ext_reset_op);
6387 register_sysrq_key('D', &sysrq_sched_ext_dump_op);
6388 INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn);
6389 }
6390
6391
6392 /********************************************************************************
6393 * Helpers that can be called from the BPF scheduler.
6394 */
6395 #include <linux/btf_ids.h>
6396
6397 __bpf_kfunc_start_defs();
6398
check_builtin_idle_enabled(void)6399 static bool check_builtin_idle_enabled(void)
6400 {
6401 if (static_branch_likely(&scx_builtin_idle_enabled))
6402 return true;
6403
6404 scx_ops_error("built-in idle tracking is disabled");
6405 return false;
6406 }
6407
6408 /**
6409 * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu()
6410 * @p: task_struct to select a CPU for
6411 * @prev_cpu: CPU @p was on previously
6412 * @wake_flags: %SCX_WAKE_* flags
6413 * @is_idle: out parameter indicating whether the returned CPU is idle
6414 *
6415 * Can only be called from ops.select_cpu() if the built-in CPU selection is
6416 * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set.
6417 * @p, @prev_cpu and @wake_flags match ops.select_cpu().
6418 *
6419 * Returns the picked CPU with *@is_idle indicating whether the picked CPU is
6420 * currently idle and thus a good candidate for direct dispatching.
6421 */
scx_bpf_select_cpu_dfl(struct task_struct * p,s32 prev_cpu,u64 wake_flags,bool * is_idle)6422 __bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
6423 u64 wake_flags, bool *is_idle)
6424 {
6425 if (!ops_cpu_valid(prev_cpu, NULL))
6426 goto prev_cpu;
6427
6428 if (!check_builtin_idle_enabled())
6429 goto prev_cpu;
6430
6431 if (!scx_kf_allowed(SCX_KF_SELECT_CPU))
6432 goto prev_cpu;
6433
6434 #ifdef CONFIG_SMP
6435 return scx_select_cpu_dfl(p, prev_cpu, wake_flags, is_idle);
6436 #endif
6437
6438 prev_cpu:
6439 *is_idle = false;
6440 return prev_cpu;
6441 }
6442
6443 __bpf_kfunc_end_defs();
6444
6445 BTF_KFUNCS_START(scx_kfunc_ids_select_cpu)
6446 BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_RCU)
6447 BTF_KFUNCS_END(scx_kfunc_ids_select_cpu)
6448
6449 static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = {
6450 .owner = THIS_MODULE,
6451 .set = &scx_kfunc_ids_select_cpu,
6452 };
6453
scx_dsq_insert_preamble(struct task_struct * p,u64 enq_flags)6454 static bool scx_dsq_insert_preamble(struct task_struct *p, u64 enq_flags)
6455 {
6456 if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH))
6457 return false;
6458
6459 lockdep_assert_irqs_disabled();
6460
6461 if (unlikely(!p)) {
6462 scx_ops_error("called with NULL task");
6463 return false;
6464 }
6465
6466 if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
6467 scx_ops_error("invalid enq_flags 0x%llx", enq_flags);
6468 return false;
6469 }
6470
6471 return true;
6472 }
6473
scx_dsq_insert_commit(struct task_struct * p,u64 dsq_id,u64 enq_flags)6474 static void scx_dsq_insert_commit(struct task_struct *p, u64 dsq_id,
6475 u64 enq_flags)
6476 {
6477 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
6478 struct task_struct *ddsp_task;
6479
6480 ddsp_task = __this_cpu_read(direct_dispatch_task);
6481 if (ddsp_task) {
6482 mark_direct_dispatch(ddsp_task, p, dsq_id, enq_flags);
6483 return;
6484 }
6485
6486 if (unlikely(dspc->cursor >= scx_dsp_max_batch)) {
6487 scx_ops_error("dispatch buffer overflow");
6488 return;
6489 }
6490
6491 dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){
6492 .task = p,
6493 .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK,
6494 .dsq_id = dsq_id,
6495 .enq_flags = enq_flags,
6496 };
6497 }
6498
6499 __bpf_kfunc_start_defs();
6500
6501 /**
6502 * scx_bpf_dsq_insert - Insert a task into the FIFO queue of a DSQ
6503 * @p: task_struct to insert
6504 * @dsq_id: DSQ to insert into
6505 * @slice: duration @p can run for in nsecs, 0 to keep the current value
6506 * @enq_flags: SCX_ENQ_*
6507 *
6508 * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to
6509 * call this function spuriously. Can be called from ops.enqueue(),
6510 * ops.select_cpu(), and ops.dispatch().
6511 *
6512 * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch
6513 * and @p must match the task being enqueued.
6514 *
6515 * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p
6516 * will be directly inserted into the corresponding dispatch queue after
6517 * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be
6518 * inserted into the local DSQ of the CPU returned by ops.select_cpu().
6519 * @enq_flags are OR'd with the enqueue flags on the enqueue path before the
6520 * task is inserted.
6521 *
6522 * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
6523 * and this function can be called upto ops.dispatch_max_batch times to insert
6524 * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
6525 * remaining slots. scx_bpf_consume() flushes the batch and resets the counter.
6526 *
6527 * This function doesn't have any locking restrictions and may be called under
6528 * BPF locks (in the future when BPF introduces more flexible locking).
6529 *
6530 * @p is allowed to run for @slice. The scheduling path is triggered on slice
6531 * exhaustion. If zero, the current residual slice is maintained. If
6532 * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with
6533 * scx_bpf_kick_cpu() to trigger scheduling.
6534 */
scx_bpf_dsq_insert(struct task_struct * p,u64 dsq_id,u64 slice,u64 enq_flags)6535 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id, u64 slice,
6536 u64 enq_flags)
6537 {
6538 if (!scx_dsq_insert_preamble(p, enq_flags))
6539 return;
6540
6541 if (slice)
6542 p->scx.slice = slice;
6543 else
6544 p->scx.slice = p->scx.slice ?: 1;
6545
6546 scx_dsq_insert_commit(p, dsq_id, enq_flags);
6547 }
6548
6549 /* for backward compatibility, will be removed in v6.15 */
scx_bpf_dispatch(struct task_struct * p,u64 dsq_id,u64 slice,u64 enq_flags)6550 __bpf_kfunc void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice,
6551 u64 enq_flags)
6552 {
6553 printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch() renamed to scx_bpf_dsq_insert()");
6554 scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags);
6555 }
6556
6557 /**
6558 * scx_bpf_dsq_insert_vtime - Insert a task into the vtime priority queue of a DSQ
6559 * @p: task_struct to insert
6560 * @dsq_id: DSQ to insert into
6561 * @slice: duration @p can run for in nsecs, 0 to keep the current value
6562 * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
6563 * @enq_flags: SCX_ENQ_*
6564 *
6565 * Insert @p into the vtime priority queue of the DSQ identified by @dsq_id.
6566 * Tasks queued into the priority queue are ordered by @vtime. All other aspects
6567 * are identical to scx_bpf_dsq_insert().
6568 *
6569 * @vtime ordering is according to time_before64() which considers wrapping. A
6570 * numerically larger vtime may indicate an earlier position in the ordering and
6571 * vice-versa.
6572 *
6573 * A DSQ can only be used as a FIFO or priority queue at any given time and this
6574 * function must not be called on a DSQ which already has one or more FIFO tasks
6575 * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and
6576 * SCX_DSQ_GLOBAL) cannot be used as priority queues.
6577 */
scx_bpf_dsq_insert_vtime(struct task_struct * p,u64 dsq_id,u64 slice,u64 vtime,u64 enq_flags)6578 __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
6579 u64 slice, u64 vtime, u64 enq_flags)
6580 {
6581 if (!scx_dsq_insert_preamble(p, enq_flags))
6582 return;
6583
6584 if (slice)
6585 p->scx.slice = slice;
6586 else
6587 p->scx.slice = p->scx.slice ?: 1;
6588
6589 p->scx.dsq_vtime = vtime;
6590
6591 scx_dsq_insert_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
6592 }
6593
6594 /* for backward compatibility, will be removed in v6.15 */
scx_bpf_dispatch_vtime(struct task_struct * p,u64 dsq_id,u64 slice,u64 vtime,u64 enq_flags)6595 __bpf_kfunc void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
6596 u64 slice, u64 vtime, u64 enq_flags)
6597 {
6598 printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_vtime() renamed to scx_bpf_dsq_insert_vtime()");
6599 scx_bpf_dsq_insert_vtime(p, dsq_id, slice, vtime, enq_flags);
6600 }
6601
6602 __bpf_kfunc_end_defs();
6603
6604 BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch)
6605 BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_RCU)
6606 BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU)
6607 BTF_ID_FLAGS(func, scx_bpf_dispatch, KF_RCU)
6608 BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime, KF_RCU)
6609 BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch)
6610
6611 static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
6612 .owner = THIS_MODULE,
6613 .set = &scx_kfunc_ids_enqueue_dispatch,
6614 };
6615
scx_dsq_move(struct bpf_iter_scx_dsq_kern * kit,struct task_struct * p,u64 dsq_id,u64 enq_flags)6616 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
6617 struct task_struct *p, u64 dsq_id, u64 enq_flags)
6618 {
6619 struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;
6620 struct rq *this_rq, *src_rq, *locked_rq;
6621 bool dispatched = false;
6622 bool in_balance;
6623 unsigned long flags;
6624
6625 if (!scx_kf_allowed_if_unlocked() && !scx_kf_allowed(SCX_KF_DISPATCH))
6626 return false;
6627
6628 /*
6629 * Can be called from either ops.dispatch() locking this_rq() or any
6630 * context where no rq lock is held. If latter, lock @p's task_rq which
6631 * we'll likely need anyway.
6632 */
6633 src_rq = task_rq(p);
6634
6635 local_irq_save(flags);
6636 this_rq = this_rq();
6637 in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;
6638
6639 if (in_balance) {
6640 if (this_rq != src_rq) {
6641 raw_spin_rq_unlock(this_rq);
6642 raw_spin_rq_lock(src_rq);
6643 }
6644 } else {
6645 raw_spin_rq_lock(src_rq);
6646 }
6647
6648 /*
6649 * If the BPF scheduler keeps calling this function repeatedly, it can
6650 * cause similar live-lock conditions as consume_dispatch_q(). Insert a
6651 * breather if necessary.
6652 */
6653 scx_ops_breather(src_rq);
6654
6655 locked_rq = src_rq;
6656 raw_spin_lock(&src_dsq->lock);
6657
6658 /*
6659 * Did someone else get to it? @p could have already left $src_dsq, got
6660 * re-enqueud, or be in the process of being consumed by someone else.
6661 */
6662 if (unlikely(p->scx.dsq != src_dsq ||
6663 u32_before(kit->cursor.priv, p->scx.dsq_seq) ||
6664 p->scx.holding_cpu >= 0) ||
6665 WARN_ON_ONCE(src_rq != task_rq(p))) {
6666 raw_spin_unlock(&src_dsq->lock);
6667 goto out;
6668 }
6669
6670 /* @p is still on $src_dsq and stable, determine the destination */
6671 dst_dsq = find_dsq_for_dispatch(this_rq, dsq_id, p);
6672
6673 /*
6674 * Apply vtime and slice updates before moving so that the new time is
6675 * visible before inserting into $dst_dsq. @p is still on $src_dsq but
6676 * this is safe as we're locking it.
6677 */
6678 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
6679 p->scx.dsq_vtime = kit->vtime;
6680 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)
6681 p->scx.slice = kit->slice;
6682
6683 /* execute move */
6684 locked_rq = move_task_between_dsqs(p, enq_flags, src_dsq, dst_dsq);
6685 dispatched = true;
6686 out:
6687 if (in_balance) {
6688 if (this_rq != locked_rq) {
6689 raw_spin_rq_unlock(locked_rq);
6690 raw_spin_rq_lock(this_rq);
6691 }
6692 } else {
6693 raw_spin_rq_unlock_irqrestore(locked_rq, flags);
6694 }
6695
6696 kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE |
6697 __SCX_DSQ_ITER_HAS_VTIME);
6698 return dispatched;
6699 }
6700
6701 __bpf_kfunc_start_defs();
6702
6703 /**
6704 * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots
6705 *
6706 * Can only be called from ops.dispatch().
6707 */
scx_bpf_dispatch_nr_slots(void)6708 __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(void)
6709 {
6710 if (!scx_kf_allowed(SCX_KF_DISPATCH))
6711 return 0;
6712
6713 return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx->cursor);
6714 }
6715
6716 /**
6717 * scx_bpf_dispatch_cancel - Cancel the latest dispatch
6718 *
6719 * Cancel the latest dispatch. Can be called multiple times to cancel further
6720 * dispatches. Can only be called from ops.dispatch().
6721 */
scx_bpf_dispatch_cancel(void)6722 __bpf_kfunc void scx_bpf_dispatch_cancel(void)
6723 {
6724 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
6725
6726 if (!scx_kf_allowed(SCX_KF_DISPATCH))
6727 return;
6728
6729 if (dspc->cursor > 0)
6730 dspc->cursor--;
6731 else
6732 scx_ops_error("dispatch buffer underflow");
6733 }
6734
6735 /**
6736 * scx_bpf_dsq_move_to_local - move a task from a DSQ to the current CPU's local DSQ
6737 * @dsq_id: DSQ to move task from
6738 *
6739 * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's
6740 * local DSQ for execution. Can only be called from ops.dispatch().
6741 *
6742 * This function flushes the in-flight dispatches from scx_bpf_dsq_insert()
6743 * before trying to move from the specified DSQ. It may also grab rq locks and
6744 * thus can't be called under any BPF locks.
6745 *
6746 * Returns %true if a task has been moved, %false if there isn't any task to
6747 * move.
6748 */
scx_bpf_dsq_move_to_local(u64 dsq_id)6749 __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id)
6750 {
6751 struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
6752 struct scx_dispatch_q *dsq;
6753
6754 if (!scx_kf_allowed(SCX_KF_DISPATCH))
6755 return false;
6756
6757 flush_dispatch_buf(dspc->rq);
6758
6759 dsq = find_user_dsq(dsq_id);
6760 if (unlikely(!dsq)) {
6761 scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id);
6762 return false;
6763 }
6764
6765 if (consume_dispatch_q(dspc->rq, dsq)) {
6766 /*
6767 * A successfully consumed task can be dequeued before it starts
6768 * running while the CPU is trying to migrate other dispatched
6769 * tasks. Bump nr_tasks to tell balance_scx() to retry on empty
6770 * local DSQ.
6771 */
6772 dspc->nr_tasks++;
6773 return true;
6774 } else {
6775 return false;
6776 }
6777 }
6778
6779 /* for backward compatibility, will be removed in v6.15 */
scx_bpf_consume(u64 dsq_id)6780 __bpf_kfunc bool scx_bpf_consume(u64 dsq_id)
6781 {
6782 printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_consume() renamed to scx_bpf_dsq_move_to_local()");
6783 return scx_bpf_dsq_move_to_local(dsq_id);
6784 }
6785
6786 /**
6787 * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs
6788 * @it__iter: DSQ iterator in progress
6789 * @slice: duration the moved task can run for in nsecs
6790 *
6791 * Override the slice of the next task that will be moved from @it__iter using
6792 * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous
6793 * slice duration is kept.
6794 */
scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq * it__iter,u64 slice)6795 __bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter,
6796 u64 slice)
6797 {
6798 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
6799
6800 kit->slice = slice;
6801 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE;
6802 }
6803
6804 /* for backward compatibility, will be removed in v6.15 */
scx_bpf_dispatch_from_dsq_set_slice(struct bpf_iter_scx_dsq * it__iter,u64 slice)6805 __bpf_kfunc void scx_bpf_dispatch_from_dsq_set_slice(
6806 struct bpf_iter_scx_dsq *it__iter, u64 slice)
6807 {
6808 printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq_set_slice() renamed to scx_bpf_dsq_move_set_slice()");
6809 scx_bpf_dsq_move_set_slice(it__iter, slice);
6810 }
6811
6812 /**
6813 * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs
6814 * @it__iter: DSQ iterator in progress
6815 * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ
6816 *
6817 * Override the vtime of the next task that will be moved from @it__iter using
6818 * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice
6819 * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the
6820 * override is ignored and cleared.
6821 */
scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq * it__iter,u64 vtime)6822 __bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter,
6823 u64 vtime)
6824 {
6825 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
6826
6827 kit->vtime = vtime;
6828 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME;
6829 }
6830
6831 /* for backward compatibility, will be removed in v6.15 */
scx_bpf_dispatch_from_dsq_set_vtime(struct bpf_iter_scx_dsq * it__iter,u64 vtime)6832 __bpf_kfunc void scx_bpf_dispatch_from_dsq_set_vtime(
6833 struct bpf_iter_scx_dsq *it__iter, u64 vtime)
6834 {
6835 printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq_set_vtime() renamed to scx_bpf_dsq_move_set_vtime()");
6836 scx_bpf_dsq_move_set_vtime(it__iter, vtime);
6837 }
6838
6839 /**
6840 * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ
6841 * @it__iter: DSQ iterator in progress
6842 * @p: task to transfer
6843 * @dsq_id: DSQ to move @p to
6844 * @enq_flags: SCX_ENQ_*
6845 *
6846 * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ
6847 * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can
6848 * be the destination.
6849 *
6850 * For the transfer to be successful, @p must still be on the DSQ and have been
6851 * queued before the DSQ iteration started. This function doesn't care whether
6852 * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have
6853 * been queued before the iteration started.
6854 *
6855 * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update.
6856 *
6857 * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq
6858 * lock (e.g. BPF timers or SYSCALL programs).
6859 *
6860 * Returns %true if @p has been consumed, %false if @p had already been consumed
6861 * or dequeued.
6862 */
scx_bpf_dsq_move(struct bpf_iter_scx_dsq * it__iter,struct task_struct * p,u64 dsq_id,u64 enq_flags)6863 __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
6864 struct task_struct *p, u64 dsq_id,
6865 u64 enq_flags)
6866 {
6867 return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
6868 p, dsq_id, enq_flags);
6869 }
6870
6871 /* for backward compatibility, will be removed in v6.15 */
scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq * it__iter,struct task_struct * p,u64 dsq_id,u64 enq_flags)6872 __bpf_kfunc bool scx_bpf_dispatch_from_dsq(struct bpf_iter_scx_dsq *it__iter,
6873 struct task_struct *p, u64 dsq_id,
6874 u64 enq_flags)
6875 {
6876 printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq() renamed to scx_bpf_dsq_move()");
6877 return scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags);
6878 }
6879
6880 /**
6881 * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ
6882 * @it__iter: DSQ iterator in progress
6883 * @p: task to transfer
6884 * @dsq_id: DSQ to move @p to
6885 * @enq_flags: SCX_ENQ_*
6886 *
6887 * Transfer @p which is on the DSQ currently iterated by @it__iter to the
6888 * priority queue of the DSQ specified by @dsq_id. The destination must be a
6889 * user DSQ as only user DSQs support priority queue.
6890 *
6891 * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice()
6892 * and scx_bpf_dsq_move_set_vtime() to update.
6893 *
6894 * All other aspects are identical to scx_bpf_dsq_move(). See
6895 * scx_bpf_dsq_insert_vtime() for more information on @vtime.
6896 */
scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq * it__iter,struct task_struct * p,u64 dsq_id,u64 enq_flags)6897 __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter,
6898 struct task_struct *p, u64 dsq_id,
6899 u64 enq_flags)
6900 {
6901 return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
6902 p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
6903 }
6904
6905 /* for backward compatibility, will be removed in v6.15 */
scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq * it__iter,struct task_struct * p,u64 dsq_id,u64 enq_flags)6906 __bpf_kfunc bool scx_bpf_dispatch_vtime_from_dsq(struct bpf_iter_scx_dsq *it__iter,
6907 struct task_struct *p, u64 dsq_id,
6908 u64 enq_flags)
6909 {
6910 printk_deferred_once(KERN_WARNING "sched_ext: scx_bpf_dispatch_from_dsq_vtime() renamed to scx_bpf_dsq_move_vtime()");
6911 return scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags);
6912 }
6913
6914 __bpf_kfunc_end_defs();
6915
6916 BTF_KFUNCS_START(scx_kfunc_ids_dispatch)
6917 BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots)
6918 BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel)
6919 BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local)
6920 BTF_ID_FLAGS(func, scx_bpf_consume)
6921 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice)
6922 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime)
6923 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
6924 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
6925 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_slice)
6926 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_vtime)
6927 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq, KF_RCU)
6928 BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime_from_dsq, KF_RCU)
6929 BTF_KFUNCS_END(scx_kfunc_ids_dispatch)
6930
6931 static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = {
6932 .owner = THIS_MODULE,
6933 .set = &scx_kfunc_ids_dispatch,
6934 };
6935
6936 __bpf_kfunc_start_defs();
6937
6938 /**
6939 * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
6940 *
6941 * Iterate over all of the tasks currently enqueued on the local DSQ of the
6942 * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of
6943 * processed tasks. Can only be called from ops.cpu_release().
6944 */
scx_bpf_reenqueue_local(void)6945 __bpf_kfunc u32 scx_bpf_reenqueue_local(void)
6946 {
6947 LIST_HEAD(tasks);
6948 u32 nr_enqueued = 0;
6949 struct rq *rq;
6950 struct task_struct *p, *n;
6951
6952 if (!scx_kf_allowed(SCX_KF_CPU_RELEASE))
6953 return 0;
6954
6955 rq = cpu_rq(smp_processor_id());
6956 lockdep_assert_rq_held(rq);
6957
6958 /*
6959 * The BPF scheduler may choose to dispatch tasks back to
6960 * @rq->scx.local_dsq. Move all candidate tasks off to a private list
6961 * first to avoid processing the same tasks repeatedly.
6962 */
6963 list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
6964 scx.dsq_list.node) {
6965 /*
6966 * If @p is being migrated, @p's current CPU may not agree with
6967 * its allowed CPUs and the migration_cpu_stop is about to
6968 * deactivate and re-activate @p anyway. Skip re-enqueueing.
6969 *
6970 * While racing sched property changes may also dequeue and
6971 * re-enqueue a migrating task while its current CPU and allowed
6972 * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
6973 * the current local DSQ for running tasks and thus are not
6974 * visible to the BPF scheduler.
6975 */
6976 if (p->migration_pending)
6977 continue;
6978
6979 dispatch_dequeue(rq, p);
6980 list_add_tail(&p->scx.dsq_list.node, &tasks);
6981 }
6982
6983 list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
6984 list_del_init(&p->scx.dsq_list.node);
6985 do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
6986 nr_enqueued++;
6987 }
6988
6989 return nr_enqueued;
6990 }
6991
6992 __bpf_kfunc_end_defs();
6993
6994 BTF_KFUNCS_START(scx_kfunc_ids_cpu_release)
6995 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local)
6996 BTF_KFUNCS_END(scx_kfunc_ids_cpu_release)
6997
6998 static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = {
6999 .owner = THIS_MODULE,
7000 .set = &scx_kfunc_ids_cpu_release,
7001 };
7002
7003 __bpf_kfunc_start_defs();
7004
7005 /**
7006 * scx_bpf_create_dsq - Create a custom DSQ
7007 * @dsq_id: DSQ to create
7008 * @node: NUMA node to allocate from
7009 *
7010 * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable
7011 * scx callback, and any BPF_PROG_TYPE_SYSCALL prog.
7012 */
scx_bpf_create_dsq(u64 dsq_id,s32 node)7013 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node)
7014 {
7015 if (unlikely(node >= (int)nr_node_ids ||
7016 (node < 0 && node != NUMA_NO_NODE)))
7017 return -EINVAL;
7018 return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node));
7019 }
7020
7021 __bpf_kfunc_end_defs();
7022
7023 BTF_KFUNCS_START(scx_kfunc_ids_unlocked)
7024 BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_SLEEPABLE)
7025 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice)
7026 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime)
7027 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
7028 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
7029 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_slice)
7030 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq_set_vtime)
7031 BTF_ID_FLAGS(func, scx_bpf_dispatch_from_dsq, KF_RCU)
7032 BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime_from_dsq, KF_RCU)
7033 BTF_KFUNCS_END(scx_kfunc_ids_unlocked)
7034
7035 static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = {
7036 .owner = THIS_MODULE,
7037 .set = &scx_kfunc_ids_unlocked,
7038 };
7039
7040 __bpf_kfunc_start_defs();
7041
7042 /**
7043 * scx_bpf_kick_cpu - Trigger reschedule on a CPU
7044 * @cpu: cpu to kick
7045 * @flags: %SCX_KICK_* flags
7046 *
7047 * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or
7048 * trigger rescheduling on a busy CPU. This can be called from any online
7049 * scx_ops operation and the actual kicking is performed asynchronously through
7050 * an irq work.
7051 */
scx_bpf_kick_cpu(s32 cpu,u64 flags)7052 __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags)
7053 {
7054 struct rq *this_rq;
7055 unsigned long irq_flags;
7056
7057 if (!ops_cpu_valid(cpu, NULL))
7058 return;
7059
7060 local_irq_save(irq_flags);
7061
7062 this_rq = this_rq();
7063
7064 /*
7065 * While bypassing for PM ops, IRQ handling may not be online which can
7066 * lead to irq_work_queue() malfunction such as infinite busy wait for
7067 * IRQ status update. Suppress kicking.
7068 */
7069 if (scx_rq_bypassing(this_rq))
7070 goto out;
7071
7072 /*
7073 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting
7074 * rq locks. We can probably be smarter and avoid bouncing if called
7075 * from ops which don't hold a rq lock.
7076 */
7077 if (flags & SCX_KICK_IDLE) {
7078 struct rq *target_rq = cpu_rq(cpu);
7079
7080 if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT)))
7081 scx_ops_error("PREEMPT/WAIT cannot be used with SCX_KICK_IDLE");
7082
7083 if (raw_spin_rq_trylock(target_rq)) {
7084 if (can_skip_idle_kick(target_rq)) {
7085 raw_spin_rq_unlock(target_rq);
7086 goto out;
7087 }
7088 raw_spin_rq_unlock(target_rq);
7089 }
7090 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle);
7091 } else {
7092 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick);
7093
7094 if (flags & SCX_KICK_PREEMPT)
7095 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt);
7096 if (flags & SCX_KICK_WAIT)
7097 cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait);
7098 }
7099
7100 irq_work_queue(&this_rq->scx.kick_cpus_irq_work);
7101 out:
7102 local_irq_restore(irq_flags);
7103 }
7104
7105 /**
7106 * scx_bpf_dsq_nr_queued - Return the number of queued tasks
7107 * @dsq_id: id of the DSQ
7108 *
7109 * Return the number of tasks in the DSQ matching @dsq_id. If not found,
7110 * -%ENOENT is returned.
7111 */
scx_bpf_dsq_nr_queued(u64 dsq_id)7112 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
7113 {
7114 struct scx_dispatch_q *dsq;
7115 s32 ret;
7116
7117 preempt_disable();
7118
7119 if (dsq_id == SCX_DSQ_LOCAL) {
7120 ret = READ_ONCE(this_rq()->scx.local_dsq.nr);
7121 goto out;
7122 } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
7123 s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
7124
7125 if (ops_cpu_valid(cpu, NULL)) {
7126 ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr);
7127 goto out;
7128 }
7129 } else {
7130 dsq = find_user_dsq(dsq_id);
7131 if (dsq) {
7132 ret = READ_ONCE(dsq->nr);
7133 goto out;
7134 }
7135 }
7136 ret = -ENOENT;
7137 out:
7138 preempt_enable();
7139 return ret;
7140 }
7141
7142 /**
7143 * scx_bpf_destroy_dsq - Destroy a custom DSQ
7144 * @dsq_id: DSQ to destroy
7145 *
7146 * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
7147 * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
7148 * empty and no further tasks are dispatched to it. Ignored if called on a DSQ
7149 * which doesn't exist. Can be called from any online scx_ops operations.
7150 */
scx_bpf_destroy_dsq(u64 dsq_id)7151 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id)
7152 {
7153 destroy_dsq(dsq_id);
7154 }
7155
7156 /**
7157 * bpf_iter_scx_dsq_new - Create a DSQ iterator
7158 * @it: iterator to initialize
7159 * @dsq_id: DSQ to iterate
7160 * @flags: %SCX_DSQ_ITER_*
7161 *
7162 * Initialize BPF iterator @it which can be used with bpf_for_each() to walk
7163 * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes
7164 * tasks which are already queued when this function is invoked.
7165 */
bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq * it,u64 dsq_id,u64 flags)7166 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
7167 u64 flags)
7168 {
7169 struct bpf_iter_scx_dsq_kern *kit = (void *)it;
7170
7171 BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) >
7172 sizeof(struct bpf_iter_scx_dsq));
7173 BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) !=
7174 __alignof__(struct bpf_iter_scx_dsq));
7175
7176 if (flags & ~__SCX_DSQ_ITER_USER_FLAGS)
7177 return -EINVAL;
7178
7179 kit->dsq = find_user_dsq(dsq_id);
7180 if (!kit->dsq)
7181 return -ENOENT;
7182
7183 INIT_LIST_HEAD(&kit->cursor.node);
7184 kit->cursor.flags = SCX_DSQ_LNODE_ITER_CURSOR | flags;
7185 kit->cursor.priv = READ_ONCE(kit->dsq->seq);
7186
7187 return 0;
7188 }
7189
7190 /**
7191 * bpf_iter_scx_dsq_next - Progress a DSQ iterator
7192 * @it: iterator to progress
7193 *
7194 * Return the next task. See bpf_iter_scx_dsq_new().
7195 */
bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq * it)7196 __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it)
7197 {
7198 struct bpf_iter_scx_dsq_kern *kit = (void *)it;
7199 bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV;
7200 struct task_struct *p;
7201 unsigned long flags;
7202
7203 if (!kit->dsq)
7204 return NULL;
7205
7206 raw_spin_lock_irqsave(&kit->dsq->lock, flags);
7207
7208 if (list_empty(&kit->cursor.node))
7209 p = NULL;
7210 else
7211 p = container_of(&kit->cursor, struct task_struct, scx.dsq_list);
7212
7213 /*
7214 * Only tasks which were queued before the iteration started are
7215 * visible. This bounds BPF iterations and guarantees that vtime never
7216 * jumps in the other direction while iterating.
7217 */
7218 do {
7219 p = nldsq_next_task(kit->dsq, p, rev);
7220 } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq)));
7221
7222 if (p) {
7223 if (rev)
7224 list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node);
7225 else
7226 list_move(&kit->cursor.node, &p->scx.dsq_list.node);
7227 } else {
7228 list_del_init(&kit->cursor.node);
7229 }
7230
7231 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
7232
7233 return p;
7234 }
7235
7236 /**
7237 * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator
7238 * @it: iterator to destroy
7239 *
7240 * Undo scx_iter_scx_dsq_new().
7241 */
bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq * it)7242 __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
7243 {
7244 struct bpf_iter_scx_dsq_kern *kit = (void *)it;
7245
7246 if (!kit->dsq)
7247 return;
7248
7249 if (!list_empty(&kit->cursor.node)) {
7250 unsigned long flags;
7251
7252 raw_spin_lock_irqsave(&kit->dsq->lock, flags);
7253 list_del_init(&kit->cursor.node);
7254 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
7255 }
7256 kit->dsq = NULL;
7257 }
7258
7259 __bpf_kfunc_end_defs();
7260
__bstr_format(u64 * data_buf,char * line_buf,size_t line_size,char * fmt,unsigned long long * data,u32 data__sz)7261 static s32 __bstr_format(u64 *data_buf, char *line_buf, size_t line_size,
7262 char *fmt, unsigned long long *data, u32 data__sz)
7263 {
7264 struct bpf_bprintf_data bprintf_data = { .get_bin_args = true };
7265 s32 ret;
7266
7267 if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 ||
7268 (data__sz && !data)) {
7269 scx_ops_error("invalid data=%p and data__sz=%u",
7270 (void *)data, data__sz);
7271 return -EINVAL;
7272 }
7273
7274 ret = copy_from_kernel_nofault(data_buf, data, data__sz);
7275 if (ret < 0) {
7276 scx_ops_error("failed to read data fields (%d)", ret);
7277 return ret;
7278 }
7279
7280 ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8,
7281 &bprintf_data);
7282 if (ret < 0) {
7283 scx_ops_error("format preparation failed (%d)", ret);
7284 return ret;
7285 }
7286
7287 ret = bstr_printf(line_buf, line_size, fmt,
7288 bprintf_data.bin_args);
7289 bpf_bprintf_cleanup(&bprintf_data);
7290 if (ret < 0) {
7291 scx_ops_error("(\"%s\", %p, %u) failed to format",
7292 fmt, data, data__sz);
7293 return ret;
7294 }
7295
7296 return ret;
7297 }
7298
bstr_format(struct scx_bstr_buf * buf,char * fmt,unsigned long long * data,u32 data__sz)7299 static s32 bstr_format(struct scx_bstr_buf *buf,
7300 char *fmt, unsigned long long *data, u32 data__sz)
7301 {
7302 return __bstr_format(buf->data, buf->line, sizeof(buf->line),
7303 fmt, data, data__sz);
7304 }
7305
7306 __bpf_kfunc_start_defs();
7307
7308 /**
7309 * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler.
7310 * @exit_code: Exit value to pass to user space via struct scx_exit_info.
7311 * @fmt: error message format string
7312 * @data: format string parameters packaged using ___bpf_fill() macro
7313 * @data__sz: @data len, must end in '__sz' for the verifier
7314 *
7315 * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops
7316 * disabling.
7317 */
scx_bpf_exit_bstr(s64 exit_code,char * fmt,unsigned long long * data,u32 data__sz)7318 __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
7319 unsigned long long *data, u32 data__sz)
7320 {
7321 unsigned long flags;
7322
7323 raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
7324 if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
7325 scx_ops_exit_kind(SCX_EXIT_UNREG_BPF, exit_code, "%s",
7326 scx_exit_bstr_buf.line);
7327 raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
7328 }
7329
7330 /**
7331 * scx_bpf_error_bstr - Indicate fatal error
7332 * @fmt: error message format string
7333 * @data: format string parameters packaged using ___bpf_fill() macro
7334 * @data__sz: @data len, must end in '__sz' for the verifier
7335 *
7336 * Indicate that the BPF scheduler encountered a fatal error and initiate ops
7337 * disabling.
7338 */
scx_bpf_error_bstr(char * fmt,unsigned long long * data,u32 data__sz)7339 __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
7340 u32 data__sz)
7341 {
7342 unsigned long flags;
7343
7344 raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
7345 if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
7346 scx_ops_exit_kind(SCX_EXIT_ERROR_BPF, 0, "%s",
7347 scx_exit_bstr_buf.line);
7348 raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
7349 }
7350
7351 /**
7352 * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler
7353 * @fmt: format string
7354 * @data: format string parameters packaged using ___bpf_fill() macro
7355 * @data__sz: @data len, must end in '__sz' for the verifier
7356 *
7357 * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and
7358 * dump_task() to generate extra debug dump specific to the BPF scheduler.
7359 *
7360 * The extra dump may be multiple lines. A single line may be split over
7361 * multiple calls. The last line is automatically terminated.
7362 */
scx_bpf_dump_bstr(char * fmt,unsigned long long * data,u32 data__sz)7363 __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
7364 u32 data__sz)
7365 {
7366 struct scx_dump_data *dd = &scx_dump_data;
7367 struct scx_bstr_buf *buf = &dd->buf;
7368 s32 ret;
7369
7370 if (raw_smp_processor_id() != dd->cpu) {
7371 scx_ops_error("scx_bpf_dump() must only be called from ops.dump() and friends");
7372 return;
7373 }
7374
7375 /* append the formatted string to the line buf */
7376 ret = __bstr_format(buf->data, buf->line + dd->cursor,
7377 sizeof(buf->line) - dd->cursor, fmt, data, data__sz);
7378 if (ret < 0) {
7379 dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)",
7380 dd->prefix, fmt, data, data__sz, ret);
7381 return;
7382 }
7383
7384 dd->cursor += ret;
7385 dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line));
7386
7387 if (!dd->cursor)
7388 return;
7389
7390 /*
7391 * If the line buf overflowed or ends in a newline, flush it into the
7392 * dump. This is to allow the caller to generate a single line over
7393 * multiple calls. As ops_dump_flush() can also handle multiple lines in
7394 * the line buf, the only case which can lead to an unexpected
7395 * truncation is when the caller keeps generating newlines in the middle
7396 * instead of the end consecutively. Don't do that.
7397 */
7398 if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n')
7399 ops_dump_flush();
7400 }
7401
7402 /**
7403 * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU
7404 * @cpu: CPU of interest
7405 *
7406 * Return the maximum relative capacity of @cpu in relation to the most
7407 * performant CPU in the system. The return value is in the range [1,
7408 * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur().
7409 */
scx_bpf_cpuperf_cap(s32 cpu)7410 __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu)
7411 {
7412 if (ops_cpu_valid(cpu, NULL))
7413 return arch_scale_cpu_capacity(cpu);
7414 else
7415 return SCX_CPUPERF_ONE;
7416 }
7417
7418 /**
7419 * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU
7420 * @cpu: CPU of interest
7421 *
7422 * Return the current relative performance of @cpu in relation to its maximum.
7423 * The return value is in the range [1, %SCX_CPUPERF_ONE].
7424 *
7425 * The current performance level of a CPU in relation to the maximum performance
7426 * available in the system can be calculated as follows:
7427 *
7428 * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE
7429 *
7430 * The result is in the range [1, %SCX_CPUPERF_ONE].
7431 */
scx_bpf_cpuperf_cur(s32 cpu)7432 __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu)
7433 {
7434 if (ops_cpu_valid(cpu, NULL))
7435 return arch_scale_freq_capacity(cpu);
7436 else
7437 return SCX_CPUPERF_ONE;
7438 }
7439
7440 /**
7441 * scx_bpf_cpuperf_set - Set the relative performance target of a CPU
7442 * @cpu: CPU of interest
7443 * @perf: target performance level [0, %SCX_CPUPERF_ONE]
7444 *
7445 * Set the target performance level of @cpu to @perf. @perf is in linear
7446 * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the
7447 * schedutil cpufreq governor chooses the target frequency.
7448 *
7449 * The actual performance level chosen, CPU grouping, and the overhead and
7450 * latency of the operations are dependent on the hardware and cpufreq driver in
7451 * use. Consult hardware and cpufreq documentation for more information. The
7452 * current performance level can be monitored using scx_bpf_cpuperf_cur().
7453 */
scx_bpf_cpuperf_set(s32 cpu,u32 perf)7454 __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf)
7455 {
7456 if (unlikely(perf > SCX_CPUPERF_ONE)) {
7457 scx_ops_error("Invalid cpuperf target %u for CPU %d", perf, cpu);
7458 return;
7459 }
7460
7461 if (ops_cpu_valid(cpu, NULL)) {
7462 struct rq *rq = cpu_rq(cpu);
7463
7464 rq->scx.cpuperf_target = perf;
7465
7466 rcu_read_lock_sched_notrace();
7467 cpufreq_update_util(cpu_rq(cpu), 0);
7468 rcu_read_unlock_sched_notrace();
7469 }
7470 }
7471
7472 /**
7473 * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs
7474 *
7475 * All valid CPU IDs in the system are smaller than the returned value.
7476 */
scx_bpf_nr_cpu_ids(void)7477 __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void)
7478 {
7479 return nr_cpu_ids;
7480 }
7481
7482 /**
7483 * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask
7484 */
scx_bpf_get_possible_cpumask(void)7485 __bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void)
7486 {
7487 return cpu_possible_mask;
7488 }
7489
7490 /**
7491 * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask
7492 */
scx_bpf_get_online_cpumask(void)7493 __bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void)
7494 {
7495 return cpu_online_mask;
7496 }
7497
7498 /**
7499 * scx_bpf_put_cpumask - Release a possible/online cpumask
7500 * @cpumask: cpumask to release
7501 */
scx_bpf_put_cpumask(const struct cpumask * cpumask)7502 __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask)
7503 {
7504 /*
7505 * Empty function body because we aren't actually acquiring or releasing
7506 * a reference to a global cpumask, which is read-only in the caller and
7507 * is never released. The acquire / release semantics here are just used
7508 * to make the cpumask is a trusted pointer in the caller.
7509 */
7510 }
7511
7512 /**
7513 * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking
7514 * per-CPU cpumask.
7515 *
7516 * Returns NULL if idle tracking is not enabled, or running on a UP kernel.
7517 */
scx_bpf_get_idle_cpumask(void)7518 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(void)
7519 {
7520 if (!check_builtin_idle_enabled())
7521 return cpu_none_mask;
7522
7523 #ifdef CONFIG_SMP
7524 return idle_masks.cpu;
7525 #else
7526 return cpu_none_mask;
7527 #endif
7528 }
7529
7530 /**
7531 * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking,
7532 * per-physical-core cpumask. Can be used to determine if an entire physical
7533 * core is free.
7534 *
7535 * Returns NULL if idle tracking is not enabled, or running on a UP kernel.
7536 */
scx_bpf_get_idle_smtmask(void)7537 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(void)
7538 {
7539 if (!check_builtin_idle_enabled())
7540 return cpu_none_mask;
7541
7542 #ifdef CONFIG_SMP
7543 if (sched_smt_active())
7544 return idle_masks.smt;
7545 else
7546 return idle_masks.cpu;
7547 #else
7548 return cpu_none_mask;
7549 #endif
7550 }
7551
7552 /**
7553 * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to
7554 * either the percpu, or SMT idle-tracking cpumask.
7555 * @idle_mask: &cpumask to use
7556 */
scx_bpf_put_idle_cpumask(const struct cpumask * idle_mask)7557 __bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask)
7558 {
7559 /*
7560 * Empty function body because we aren't actually acquiring or releasing
7561 * a reference to a global idle cpumask, which is read-only in the
7562 * caller and is never released. The acquire / release semantics here
7563 * are just used to make the cpumask a trusted pointer in the caller.
7564 */
7565 }
7566
7567 /**
7568 * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state
7569 * @cpu: cpu to test and clear idle for
7570 *
7571 * Returns %true if @cpu was idle and its idle state was successfully cleared.
7572 * %false otherwise.
7573 *
7574 * Unavailable if ops.update_idle() is implemented and
7575 * %SCX_OPS_KEEP_BUILTIN_IDLE is not set.
7576 */
scx_bpf_test_and_clear_cpu_idle(s32 cpu)7577 __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu)
7578 {
7579 if (!check_builtin_idle_enabled())
7580 return false;
7581
7582 if (ops_cpu_valid(cpu, NULL))
7583 return test_and_clear_cpu_idle(cpu);
7584 else
7585 return false;
7586 }
7587
7588 /**
7589 * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu
7590 * @cpus_allowed: Allowed cpumask
7591 * @flags: %SCX_PICK_IDLE_CPU_* flags
7592 *
7593 * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu
7594 * number on success. -%EBUSY if no matching cpu was found.
7595 *
7596 * Idle CPU tracking may race against CPU scheduling state transitions. For
7597 * example, this function may return -%EBUSY as CPUs are transitioning into the
7598 * idle state. If the caller then assumes that there will be dispatch events on
7599 * the CPUs as they were all busy, the scheduler may end up stalling with CPUs
7600 * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and
7601 * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch
7602 * event in the near future.
7603 *
7604 * Unavailable if ops.update_idle() is implemented and
7605 * %SCX_OPS_KEEP_BUILTIN_IDLE is not set.
7606 */
scx_bpf_pick_idle_cpu(const struct cpumask * cpus_allowed,u64 flags)7607 __bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed,
7608 u64 flags)
7609 {
7610 if (!check_builtin_idle_enabled())
7611 return -EBUSY;
7612
7613 return scx_pick_idle_cpu(cpus_allowed, flags);
7614 }
7615
7616 /**
7617 * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU
7618 * @cpus_allowed: Allowed cpumask
7619 * @flags: %SCX_PICK_IDLE_CPU_* flags
7620 *
7621 * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any
7622 * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu
7623 * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is
7624 * empty.
7625 *
7626 * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not
7627 * set, this function can't tell which CPUs are idle and will always pick any
7628 * CPU.
7629 */
scx_bpf_pick_any_cpu(const struct cpumask * cpus_allowed,u64 flags)7630 __bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed,
7631 u64 flags)
7632 {
7633 s32 cpu;
7634
7635 if (static_branch_likely(&scx_builtin_idle_enabled)) {
7636 cpu = scx_pick_idle_cpu(cpus_allowed, flags);
7637 if (cpu >= 0)
7638 return cpu;
7639 }
7640
7641 cpu = cpumask_any_distribute(cpus_allowed);
7642 if (cpu < nr_cpu_ids)
7643 return cpu;
7644 else
7645 return -EBUSY;
7646 }
7647
7648 /**
7649 * scx_bpf_task_running - Is task currently running?
7650 * @p: task of interest
7651 */
scx_bpf_task_running(const struct task_struct * p)7652 __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p)
7653 {
7654 return task_rq(p)->curr == p;
7655 }
7656
7657 /**
7658 * scx_bpf_task_cpu - CPU a task is currently associated with
7659 * @p: task of interest
7660 */
scx_bpf_task_cpu(const struct task_struct * p)7661 __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p)
7662 {
7663 return task_cpu(p);
7664 }
7665
7666 /**
7667 * scx_bpf_cpu_rq - Fetch the rq of a CPU
7668 * @cpu: CPU of the rq
7669 */
scx_bpf_cpu_rq(s32 cpu)7670 __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu)
7671 {
7672 if (!ops_cpu_valid(cpu, NULL))
7673 return NULL;
7674
7675 return cpu_rq(cpu);
7676 }
7677
7678 /**
7679 * scx_bpf_task_cgroup - Return the sched cgroup of a task
7680 * @p: task of interest
7681 *
7682 * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with
7683 * from the scheduler's POV. SCX operations should use this function to
7684 * determine @p's current cgroup as, unlike following @p->cgroups,
7685 * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all
7686 * rq-locked operations. Can be called on the parameter tasks of rq-locked
7687 * operations. The restriction guarantees that @p's rq is locked by the caller.
7688 */
7689 #ifdef CONFIG_CGROUP_SCHED
scx_bpf_task_cgroup(struct task_struct * p)7690 __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p)
7691 {
7692 struct task_group *tg = p->sched_task_group;
7693 struct cgroup *cgrp = &cgrp_dfl_root.cgrp;
7694
7695 if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p))
7696 goto out;
7697
7698 cgrp = tg_cgrp(tg);
7699
7700 out:
7701 cgroup_get(cgrp);
7702 return cgrp;
7703 }
7704 #endif
7705
7706 /**
7707 * scx_bpf_now - Returns a high-performance monotonically non-decreasing
7708 * clock for the current CPU. The clock returned is in nanoseconds.
7709 *
7710 * It provides the following properties:
7711 *
7712 * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently
7713 * to account for execution time and track tasks' runtime properties.
7714 * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which
7715 * eventually reads a hardware timestamp counter -- is neither performant nor
7716 * scalable. scx_bpf_now() aims to provide a high-performance clock by
7717 * using the rq clock in the scheduler core whenever possible.
7718 *
7719 * 2) High enough resolution for the BPF scheduler use cases: In most BPF
7720 * scheduler use cases, the required clock resolution is lower than the most
7721 * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically
7722 * uses the rq clock in the scheduler core whenever it is valid. It considers
7723 * that the rq clock is valid from the time the rq clock is updated
7724 * (update_rq_clock) until the rq is unlocked (rq_unpin_lock).
7725 *
7726 * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now()
7727 * guarantees the clock never goes backward when comparing them in the same
7728 * CPU. On the other hand, when comparing clocks in different CPUs, there
7729 * is no such guarantee -- the clock can go backward. It provides a
7730 * monotonically *non-decreasing* clock so that it would provide the same
7731 * clock values in two different scx_bpf_now() calls in the same CPU
7732 * during the same period of when the rq clock is valid.
7733 */
scx_bpf_now(void)7734 __bpf_kfunc u64 scx_bpf_now(void)
7735 {
7736 struct rq *rq;
7737 u64 clock;
7738
7739 preempt_disable();
7740
7741 rq = this_rq();
7742 if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) {
7743 /*
7744 * If the rq clock is valid, use the cached rq clock.
7745 *
7746 * Note that scx_bpf_now() is re-entrant between a process
7747 * context and an interrupt context (e.g., timer interrupt).
7748 * However, we don't need to consider the race between them
7749 * because such race is not observable from a caller.
7750 */
7751 clock = READ_ONCE(rq->scx.clock);
7752 } else {
7753 /*
7754 * Otherwise, return a fresh rq clock.
7755 *
7756 * The rq clock is updated outside of the rq lock.
7757 * In this case, keep the updated rq clock invalid so the next
7758 * kfunc call outside the rq lock gets a fresh rq clock.
7759 */
7760 clock = sched_clock_cpu(cpu_of(rq));
7761 }
7762
7763 preempt_enable();
7764
7765 return clock;
7766 }
7767
7768 __bpf_kfunc_end_defs();
7769
7770 BTF_KFUNCS_START(scx_kfunc_ids_any)
7771 BTF_ID_FLAGS(func, scx_bpf_kick_cpu)
7772 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)
7773 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)
7774 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_ITER_NEW | KF_RCU_PROTECTED)
7775 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
7776 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
7777 BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_TRUSTED_ARGS)
7778 BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS)
7779 BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_TRUSTED_ARGS)
7780 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap)
7781 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur)
7782 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set)
7783 BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
7784 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
7785 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
7786 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
7787 BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE)
7788 BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE)
7789 BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)
7790 BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle)
7791 BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_RCU)
7792 BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_RCU)
7793 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)
7794 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
7795 BTF_ID_FLAGS(func, scx_bpf_cpu_rq)
7796 #ifdef CONFIG_CGROUP_SCHED
7797 BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE)
7798 #endif
7799 BTF_ID_FLAGS(func, scx_bpf_now)
7800 BTF_KFUNCS_END(scx_kfunc_ids_any)
7801
7802 static const struct btf_kfunc_id_set scx_kfunc_set_any = {
7803 .owner = THIS_MODULE,
7804 .set = &scx_kfunc_ids_any,
7805 };
7806
scx_init(void)7807 static int __init scx_init(void)
7808 {
7809 int ret;
7810
7811 /*
7812 * kfunc registration can't be done from init_sched_ext_class() as
7813 * register_btf_kfunc_id_set() needs most of the system to be up.
7814 *
7815 * Some kfuncs are context-sensitive and can only be called from
7816 * specific SCX ops. They are grouped into BTF sets accordingly.
7817 * Unfortunately, BPF currently doesn't have a way of enforcing such
7818 * restrictions. Eventually, the verifier should be able to enforce
7819 * them. For now, register them the same and make each kfunc explicitly
7820 * check using scx_kf_allowed().
7821 */
7822 if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
7823 &scx_kfunc_set_select_cpu)) ||
7824 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
7825 &scx_kfunc_set_enqueue_dispatch)) ||
7826 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
7827 &scx_kfunc_set_dispatch)) ||
7828 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
7829 &scx_kfunc_set_cpu_release)) ||
7830 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
7831 &scx_kfunc_set_unlocked)) ||
7832 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
7833 &scx_kfunc_set_unlocked)) ||
7834 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
7835 &scx_kfunc_set_any)) ||
7836 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
7837 &scx_kfunc_set_any)) ||
7838 (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
7839 &scx_kfunc_set_any))) {
7840 pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret);
7841 return ret;
7842 }
7843
7844 ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops);
7845 if (ret) {
7846 pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret);
7847 return ret;
7848 }
7849
7850 ret = register_pm_notifier(&scx_pm_notifier);
7851 if (ret) {
7852 pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret);
7853 return ret;
7854 }
7855
7856 scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj);
7857 if (!scx_kset) {
7858 pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n");
7859 return -ENOMEM;
7860 }
7861
7862 ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group);
7863 if (ret < 0) {
7864 pr_err("sched_ext: Failed to add global attributes\n");
7865 return ret;
7866 }
7867
7868 return 0;
7869 }
7870 __initcall(scx_init);
7871