xref: /linux/kernel/sched/ext/ext.c (revision 700a332c3fe33014bbd5ae852d71ba679fc29e0a)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
4  *
5  * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
6  * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
7  * Copyright (c) 2022 David Vernet <dvernet@meta.com>
8  */
9 #include <linux/bitmap.h>
10 #include <linux/btf_ids.h>
11 #include <linux/rhashtable.h>
12 #include <linux/sched/clock.h>
13 #include <linux/sched/isolation.h>
14 #include <linux/suspend.h>
15 #include <linux/sysrq.h>
16 
17 #include "../pelt.h"
18 #include "internal.h"
19 #include "cid.h"
20 #include "arena.h"
21 #include "idle.h"
22 #include "sub.h"
23 #include "inlines.h"
24 
25 DEFINE_RAW_SPINLOCK(scx_sched_lock);
26 
27 /*
28  * Block a retained proxy donor, telling put_prev_task_scx() that the donor is
29  * about to be dequeued and must not be reenqueued.
30  */
31 static void scx_block_proxy_donor(struct rq *rq, struct task_struct *p)
32 {
33 	rq->scx.flags |= SCX_RQ_PROXY_BLOCKING;
34 	sched_proxy_block_task(rq, p);
35 	rq->scx.flags &= ~SCX_RQ_PROXY_BLOCKING;
36 }
37 
38 bool __scx_allow_proxy_exec(const struct task_struct *p)
39 {
40 	struct scx_sched *sch;
41 
42 	if (p->sched_class != &ext_sched_class)
43 		return true;
44 
45 	sch = scx_task_sched(p);
46 	return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED);
47 }
48 
49 /*
50  * End retained proxy execution before changing @p's BPF scheduler ownership.
51  * Called with @p's pi and rq locks held immediately before
52  * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock
53  * is updated only once.
54  */
55 void scx_prepare_task_sched_change(struct task_struct *p)
56 {
57 	lockdep_assert_held(&p->pi_lock);
58 	lockdep_assert_rq_held(task_rq(p));
59 
60 	update_rq_clock(task_rq(p));
61 	scx_block_proxy_donor(task_rq(p), p);
62 }
63 
64 /*
65  * NOTE: sched_ext is in the process of growing multiple scheduler support and
66  * scx_root usage is in a transitional state. Naked dereferences are safe if the
67  * caller is one of the tasks attached to SCX and explicit RCU dereference is
68  * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but
69  * are used as temporary markers to indicate that the dereferences need to be
70  * updated to point to the associated scheduler instances rather than scx_root.
71  */
72 struct scx_sched __rcu *scx_root;
73 
74 /*
75  * All scheds, writers must hold both scx_enable_mutex and scx_sched_lock.
76  * Readers can hold either or rcu_read_lock().
77  */
78 LIST_HEAD(scx_sched_all);
79 
80 #ifdef CONFIG_EXT_SUB_SCHED
81 static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg,
82 					   const void *ptr)
83 {
84 	const struct scx_sched *sch = ptr;
85 
86 	BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64));
87 
88 	return sch->ops.sub_cgroup_id != *(const u64 *)arg->key;
89 }
90 
91 const struct rhashtable_params scx_sched_hash_params = {
92 	.key_len		= sizeof_field(struct scx_sched, ops.sub_cgroup_id),
93 	.key_offset		= offsetof(struct scx_sched, ops.sub_cgroup_id),
94 	.head_offset		= offsetof(struct scx_sched, hash_node),
95 	.obj_cmpfn		= scx_sched_cmpfn,
96 	.insecure_elasticity	= true,	/* inserted under scx_sched_lock */
97 };
98 
99 struct rhashtable scx_sched_hash;
100 #endif
101 
102 /* see SCX_OPS_TID_TO_TASK */
103 static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
104 {
105 	const struct sched_ext_entity *scx = ptr;
106 
107 	BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64));
108 
109 	return scx->tid != *(const u64 *)arg->key;
110 }
111 
112 static const struct rhashtable_params scx_tid_hash_params = {
113 	.key_len		= sizeof_field(struct sched_ext_entity, tid),
114 	.key_offset		= offsetof(struct sched_ext_entity, tid),
115 	.head_offset		= offsetof(struct sched_ext_entity, tid_hash_node),
116 	.obj_cmpfn		= scx_tid_cmpfn,
117 	.insecure_elasticity	= true,	/* inserted/removed under scx_tasks_lock */
118 };
119 static struct rhashtable scx_tid_hash;
120 
121 /*
122  * During exit, a task may schedule after losing its PIDs. When disabling the
123  * BPF scheduler, we need to be able to iterate tasks in every state to
124  * guarantee system safety. Maintain a dedicated task list which contains every
125  * task between its fork and eventual free.
126  */
127 static DEFINE_RAW_SPINLOCK(scx_tasks_lock);
128 static LIST_HEAD(scx_tasks);
129 
130 /* ops enable/disable */
131 DEFINE_MUTEX(scx_enable_mutex);
132 DEFINE_STATIC_KEY_FALSE(__scx_enabled);
133 DEFINE_PERCPU_RWSEM(scx_fork_rwsem);
134 static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);
135 static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
136 static bool scx_init_task_enabled;
137 static bool scx_switching_all;
138 DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
139 static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled);
140 
141 /*
142  * Gates cgroup ops delivery. Set at the end of the cgroup init phase of root
143  * enable and cleared before root disable starts tearing down tasks, both under
144  * scx_cgroup_lock(). Holding cgroup_lock() and seeing %true guarantees no race
145  * against root tearing down tasks.
146  */
147 bool scx_cgroup_enabled;
148 
149 /*
150  * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler
151  * and the tid->task table is live. Wraps the static key so callers don't
152  * take the address, and hints "likely enabled" for the common case where
153  * the feature is in use.
154  */
155 static inline bool scx_tid_to_task_enabled(void)
156 {
157 	return static_branch_likely(&__scx_tid_to_task_enabled);
158 }
159 
160 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
161 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
162 
163 /* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */
164 static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1);
165 
166 /* is @dsq synchronized by the containing rq lock instead of dsq->lock? */
167 static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq)
168 {
169 	switch (dsq->id) {
170 	case SCX_DSQ_LOCAL:
171 	case SCX_DSQ_REJECT:
172 	case SCX_DSQ_RESCUE:
173 		return true;
174 	default:
175 		return false;
176 	}
177 }
178 
179 /* Cursor for unique scx_sched instance ids. id 0 is reserved. */
180 static atomic64_t scx_sched_id_cursor = ATOMIC64_INIT(0);
181 
182 #ifdef CONFIG_EXT_SUB_SCHED
183 /*
184  * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit
185  * tasks for the sub-sched being enabled. Use a global variable instead of a
186  * per-task field as all enables are serialized.
187  */
188 struct scx_sched *scx_enabling_sub_sched;
189 #else
190 #define scx_enabling_sub_sched	(struct scx_sched *)NULL
191 #endif	/* CONFIG_EXT_SUB_SCHED */
192 
193 /*
194  * A monotonically increasing sequence number that is incremented every time a
195  * scheduler is enabled. This can be used to check if any custom sched_ext
196  * scheduler has ever been used in the system.
197  */
198 static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0);
199 
200 /*
201  * Watchdog interval. All scx_sched's share a single watchdog timer and the
202  * interval is half of the shortest sch->watchdog_timeout.
203  */
204 static unsigned long scx_watchdog_interval;
205 
206 /*
207  * The last time the delayed work was run. This delayed work relies on
208  * ksoftirqd being able to run to service timer interrupts, so it's possible
209  * that this work itself could get wedged. To account for this, we check that
210  * it's not stalled in the timer tick, and trigger an error if it is.
211  */
212 static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES;
213 
214 static struct delayed_work scx_watchdog_work;
215 
216 /*
217  * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence
218  * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu
219  * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated
220  * lazily when enabling and freed when disabling to avoid waste when sched_ext
221  * isn't active.
222  */
223 struct scx_kick_syncs {
224 	struct rcu_head		rcu;
225 	unsigned long		syncs[];
226 };
227 
228 static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs);
229 
230 /*
231  * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of
232  * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without
233  * further synchronization. See scx_alloc_tid().
234  */
235 struct scx_tid_alloc {
236 	u64	next;
237 	u64	end;
238 };
239 static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc);
240 
241 /*
242  * Direct dispatch marker.
243  *
244  * Non-NULL values are used for direct dispatch from enqueue path. A valid
245  * pointer points to the task currently being enqueued. An ERR_PTR value is used
246  * to indicate that direct dispatch has already happened.
247  */
248 static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
249 
250 static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
251 {
252 	const struct scx_dispatch_q *dsq = ptr;
253 
254 	BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64));
255 
256 	return dsq->id != *(const u64 *)arg->key;
257 }
258 
259 static const struct rhashtable_params dsq_hash_params = {
260 	.key_len		= sizeof_field(struct scx_dispatch_q, id),
261 	.key_offset		= offsetof(struct scx_dispatch_q, id),
262 	.head_offset		= offsetof(struct scx_dispatch_q, hash_node),
263 	.obj_cmpfn		= dsq_cmpfn,
264 };
265 
266 static LLIST_HEAD(dsqs_to_free);
267 
268 /* ops debug dump */
269 static DEFINE_RAW_SPINLOCK(scx_dump_lock);
270 
271 struct scx_dump_data {
272 	s32			cpu;
273 	bool			first;
274 	s32			cursor;
275 	struct seq_buf		*s;
276 	const char		*prefix;
277 	struct scx_bstr_buf	buf;
278 };
279 
280 static struct scx_dump_data scx_dump_data = {
281 	.cpu			= -1,
282 };
283 
284 /* /sys/kernel/sched_ext interface */
285 static struct kset *scx_kset;
286 
287 /*
288  * Parameters that can be adjusted through /sys/module/sched_ext/parameters.
289  * There usually is no reason to modify these as normal scheduler operation
290  * shouldn't be affected by them. The knobs are primarily for debugging.
291  */
292 static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC;
293 static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US;
294 
295 static int set_slice_us(const char *val, const struct kernel_param *kp)
296 {
297 	return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC);
298 }
299 
300 static const struct kernel_param_ops slice_us_param_ops = {
301 	.set = set_slice_us,
302 	.get = param_get_uint,
303 };
304 
305 static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp)
306 {
307 	return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC);
308 }
309 
310 static const struct kernel_param_ops bypass_lb_intv_us_param_ops = {
311 	.set = set_bypass_lb_intv_us,
312 	.get = param_get_uint,
313 };
314 
315 #undef MODULE_PARAM_PREFIX
316 #define MODULE_PARAM_PREFIX	"sched_ext."
317 
318 module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600);
319 MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)");
320 module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600);
321 MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)");
322 
323 #undef MODULE_PARAM_PREFIX
324 
325 #define CREATE_TRACE_POINTS
326 #include <trace/events/sched_ext.h>
327 
328 static void run_deferred(struct rq *rq);
329 static bool task_dead_and_done(struct task_struct *p);
330 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
331 
332 __printf(5, 6) bool __scx_exit(struct scx_sched *sch,
333 			       enum scx_exit_kind kind, s64 exit_code,
334 			       s32 exit_cpu, const char *fmt, ...)
335 {
336 	va_list args;
337 	bool ret;
338 
339 	va_start(args, fmt);
340 	ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args);
341 	va_end(args);
342 
343 	return ret;
344 }
345 
346 static long jiffies_delta_msecs(unsigned long at, unsigned long now)
347 {
348 	if (time_after(at, now))
349 		return jiffies_to_msecs(at - now);
350 	else
351 		return -(long)jiffies_to_msecs(now - at);
352 }
353 
354 static bool u32_before(u32 a, u32 b)
355 {
356 	return (s32)(a - b) < 0;
357 }
358 
359 /**
360  * scx_is_descendant - Test whether sched is a descendant
361  * @sch: sched to test
362  * @ancestor: ancestor sched to test against
363  *
364  * Test whether @sch is a descendant of @ancestor.
365  */
366 bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor)
367 {
368 	if (sch->level < ancestor->level)
369 		return false;
370 	return sch->ancestors[ancestor->level] == ancestor;
371 }
372 
373 static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu)
374 {
375 	return &sch->pnode[cpu_to_node(cpu)]->global_dsq;
376 }
377 
378 static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id)
379 {
380 	return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params);
381 }
382 
383 static const struct sched_class *scx_setscheduler_class(struct task_struct *p)
384 {
385 	if (p->sched_class == &stop_sched_class)
386 		return &stop_sched_class;
387 
388 	return __setscheduler_class(p->policy, p->prio);
389 }
390 
391 static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, struct rq *rq,
392 						    struct task_struct *p, u64 *enq_flags)
393 {
394 	struct scx_sched *host = scx_exec_sched(sch, cpu_of(rq));
395 	struct scx_dispatch_q *dsq;
396 
397 	/*
398 	 * If @sch is a sub-sched which is bypassing, its tasks should go into
399 	 * the bypass DSQs of the nearest ancestor which is not bypassing. The
400 	 * not-bypassing ancestor is responsible for scheduling all tasks from
401 	 * bypassing sub-trees. If all ancestors including root are bypassing,
402 	 * all tasks should go to the root's bypass DSQs.
403 	 *
404 	 * Whenever a sched starts bypassing, all runnable tasks in its subtree
405 	 * are re-enqueued after scx_bypassing() is turned on, guaranteeing that
406 	 * all tasks are transferred to the right DSQs.
407 	 *
408 	 * A cpu's dispatch only descends into a host holding %SCX_CAP_BASE on
409 	 * it, so the bypass DSQ of a host without BASE here is never consumed.
410 	 * Send @p to rescue on this cpu instead, resolving against @host.
411 	 * scx_dispatch_enqueue() resolves a local DSQ again, against the
412 	 * bypassing @sch. %SCX_ENQ_IGNORE_CAPS makes that second resolution
413 	 * admit @p.
414 	 *
415 	 * A migration-disabled task can't leave this cpu, and a task with a
416 	 * migration pending must run here until the migration moves it. Either
417 	 * would wait on the unconsumed bypass DSQ until the host regains BASE.
418 	 * Resolve them too when rescue is disabled. The resolution admits both
419 	 * to the local DSQ, as it does any insert of such a task.
420 	 */
421 	if (unlikely(scx_missing_caps(host, cpu_of(rq), SCX_CAP_BASE)) &&
422 	    (scx_rescue_enabled() || is_migration_disabled(p) || p->migration_pending)) {
423 		*enq_flags |= SCX_ENQ_RESCUE;
424 		dsq = scx_resolve_local_dsq(host, rq, p, enq_flags);
425 		*enq_flags |= SCX_ENQ_IGNORE_CAPS;
426 	} else {
427 		dsq = scx_bypass_dsq(host, cpu_of(rq));
428 	}
429 	return dsq;
430 }
431 
432 /**
433  * rq_is_open - Is the rq available for immediate execution of an SCX task?
434  * @rq: rq to test
435  * @enq_flags: optional %SCX_ENQ_* of the task being enqueued
436  *
437  * Returns %true if @rq is currently open for executing an SCX task. After a
438  * %false return, @rq is guaranteed to invoke SCX dispatch path at least once
439  * before going to idle and not inserting a task into @rq's local DSQ after a
440  * %false return doesn't cause @rq to stall.
441  */
442 static bool rq_is_open(struct rq *rq, u64 enq_flags)
443 {
444 	lockdep_assert_rq_held(rq);
445 
446 	/*
447 	 * A higher-priority class task is either running or in the process of
448 	 * waking up on @rq.
449 	 */
450 	if (sched_class_above(rq->next_class, &ext_sched_class))
451 		return false;
452 
453 	/*
454 	 * @rq is either in transition to or in idle and there is no
455 	 * higher-priority class task waking up on it.
456 	 */
457 	if (sched_class_above(&ext_sched_class, rq->next_class))
458 		return true;
459 
460 	/*
461 	 * @rq is either picking, in transition to, or running an SCX task.
462 	 */
463 
464 	/*
465 	 * A local insertion can raise next_class to ext while the donor is
466 	 * still idle. Keep the rq open so that the local DSQ head is not
467 	 * unnecessarily reenqueued before the CPU can pick it.
468 	 */
469 	if (rq->donor == rq->idle)
470 		return true;
471 
472 	/*
473 	 * If we're in the dispatch path holding rq lock, $donor may or may not
474 	 * be ready depending on whether the on-going dispatch decides to extend
475 	 * $donor's slice. We say yes here and resolve it at the end of dispatch.
476 	 * See dispatch_one().
477 	 */
478 	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
479 		return true;
480 
481 	/*
482 	 * The preemption flags clear $donor's slice if on SCX and kick dispatch,
483 	 * so allow it to avoid spuriously triggering reenq on a combined
484 	 * PREEMPT|IMMED insertion.
485 	 */
486 	if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) {
487 		struct task_struct *donor = rq->donor;
488 
489 		/*
490 		 * A protected slice refuses the preemption and the cpu stays
491 		 * occupied. See rq_owned_post_enq().
492 		 */
493 		return donor->sched_class != &ext_sched_class ||
494 			likely(!(donor->scx.flags & SCX_TASK_PROTECTED));
495 	}
496 
497 	/*
498 	 * @rq is either in transition to or running an SCX task and can't go
499 	 * idle without another SCX dispatch cycle.
500 	 */
501 	return false;
502 }
503 
504 /*
505  * Track the rq currently locked.
506  *
507  * This allows kfuncs to safely operate on rq from any scx ops callback,
508  * knowing which rq is already locked.
509  */
510 DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
511 
512 /*
513  * Under core scheduling, a pick that releases the rq lock invalidates the
514  * core-wide selection it is part of. Count the releases so that the core-sched
515  * pick can tell whether one happened across dispatch.
516  */
517 static void scx_rq_lock_drop(struct rq *rq)
518 {
519 	lockdep_assert_rq_held(rq);
520 #ifdef CONFIG_SCHED_CORE
521 	if (sched_core_enabled(rq))
522 		rq->scx.lock_drop_seq++;
523 #endif
524 }
525 
526 static void switch_rq_lock(struct rq *from, struct rq *to)
527 {
528 	bool tracked = scx_locked_rq() == from;
529 
530 	if (tracked)
531 		update_locked_rq(NULL);
532 	scx_rq_lock_drop(from);
533 	raw_spin_rq_unlock(from);
534 	raw_spin_rq_lock(to);
535 	if (tracked)
536 		update_locked_rq(to);
537 }
538 
539 /*
540  * Flipped on enable per sch->is_cid_type. Declared in internal.h so
541  * subsystem inlines can read it.
542  */
543 DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
544 
545 /**
546  * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask
547  * @sch: scx_sched whose scratch to fill
548  * @cpumask: cpus to translate into cids
549  *
550  * The scratch lives in BPF-writable arena memory and its header can't be
551  * trusted, so it is rewritten from kernel geometry rather than read. Caller
552  * must hold an rq lock so this cpu is the sole kernel writer for as long as the
553  * returned address is in use.
554  */
555 static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch,
556 						const struct cpumask *cpumask)
557 {
558 	struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
559 	struct scx_cmask_ref ref;
560 
561 	scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
562 	scx_cmask_ref_from_cpumask(&ref, cpumask);
563 	return kern_va;
564 }
565 
566 /**
567  * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task
568  * @sch: scx_sched being invoked
569  * @rq: rq to update as the currently-locked rq, or NULL
570  * @task: task whose affinity is changing
571  * @cpumask: new cpumask
572  *
573  * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch
574  * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock.
575  */
576 static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq,
577 					   struct task_struct *task,
578 					   const struct cpumask *cpumask)
579 {
580 	if (scx_is_cid_type())
581 		SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task,
582 				     scx_fill_cmask_scratch(sch, cpumask));
583 	else
584 		SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask);
585 }
586 
587 enum scx_dsq_iter_flags {
588 	/* iterate in the reverse dispatch order */
589 	SCX_DSQ_ITER_REV		= 1U << 16,
590 
591 	__SCX_DSQ_ITER_DRAINED		= 1U << 28,
592 	__SCX_DSQ_ITER_HAS_REENQ	= 1U << 29,
593 	__SCX_DSQ_ITER_HAS_SLICE	= 1U << 30,
594 	__SCX_DSQ_ITER_HAS_VTIME	= 1U << 31,
595 
596 	__SCX_DSQ_ITER_USER_FLAGS	= SCX_DSQ_ITER_REV,
597 	__SCX_DSQ_ITER_ALL_FLAGS	= __SCX_DSQ_ITER_USER_FLAGS |
598 					  __SCX_DSQ_ITER_DRAINED |
599 					  __SCX_DSQ_ITER_HAS_REENQ |
600 					  __SCX_DSQ_ITER_HAS_SLICE |
601 					  __SCX_DSQ_ITER_HAS_VTIME,
602 };
603 
604 /**
605  * nldsq_next_task - Iterate to the next task in a non-local DSQ
606  * @dsq: non-local dsq being iterated
607  * @cur: current position, %NULL to start iteration
608  * @rev: walk backwards
609  *
610  * Returns %NULL when iteration is finished.
611  */
612 static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq,
613 					   struct task_struct *cur, bool rev)
614 {
615 	struct list_head *list_node;
616 	struct scx_dsq_list_node *dsq_lnode;
617 
618 	lockdep_assert_held(&dsq->lock);
619 
620 	if (cur)
621 		list_node = &cur->scx.dsq_list.node;
622 	else
623 		list_node = &dsq->list;
624 
625 	/* find the next task, need to skip BPF iteration cursors */
626 	do {
627 		if (rev)
628 			list_node = list_node->prev;
629 		else
630 			list_node = list_node->next;
631 
632 		if (list_node == &dsq->list)
633 			return NULL;
634 
635 		dsq_lnode = container_of(list_node, struct scx_dsq_list_node,
636 					 node);
637 	} while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR);
638 
639 	return container_of(dsq_lnode, struct task_struct, scx.dsq_list);
640 }
641 
642 #define nldsq_for_each_task(p, dsq)						\
643 	for ((p) = nldsq_next_task((dsq), NULL, false); (p);			\
644 	     (p) = nldsq_next_task((dsq), (p), false))
645 
646 /**
647  * nldsq_cursor_next_task - Iterate to the next task given a cursor in a non-local DSQ
648  * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR()
649  * @dsq: non-local dsq being iterated
650  *
651  * Find the next task in a cursor based iteration. The caller must have
652  * initialized @cursor using INIT_DSQ_LIST_CURSOR() and can release the DSQ lock
653  * between the iteration steps.
654  *
655  * Only tasks which were queued before @cursor was initialized are visible. This
656  * bounds the iteration and guarantees that vtime never jumps in the other
657  * direction while iterating.
658  */
659 static struct task_struct *nldsq_cursor_next_task(struct scx_dsq_list_node *cursor,
660 						  struct scx_dispatch_q *dsq)
661 {
662 	bool rev = cursor->flags & SCX_DSQ_ITER_REV;
663 	struct task_struct *p;
664 
665 	lockdep_assert_held(&dsq->lock);
666 	BUG_ON(!(cursor->flags & SCX_DSQ_LNODE_ITER_CURSOR));
667 
668 	if (list_empty(&cursor->node))
669 		p = NULL;
670 	else
671 		p = container_of(cursor, struct task_struct, scx.dsq_list);
672 
673 	/* skip cursors and tasks that were queued after @cursor init */
674 	do {
675 		p = nldsq_next_task(dsq, p, rev);
676 	} while (p && unlikely(u32_before(cursor->priv, p->scx.dsq_seq)));
677 
678 	if (p) {
679 		if (rev)
680 			list_move_tail(&cursor->node, &p->scx.dsq_list.node);
681 		else
682 			list_move(&cursor->node, &p->scx.dsq_list.node);
683 	} else {
684 		list_del_init(&cursor->node);
685 	}
686 
687 	return p;
688 }
689 
690 /**
691  * nldsq_cursor_lost_task - Test whether someone else took the task since iteration
692  * @cursor: scx_dsq_list_node initialized with INIT_DSQ_LIST_CURSOR()
693  * @rq: rq @p was on
694  * @dsq: dsq @p was on
695  * @p: target task
696  *
697  * @p is a task returned by nldsq_cursor_next_task(). The locks may have been
698  * dropped and re-acquired inbetween. Verify that no one else took or is in the
699  * process of taking @p from @dsq.
700  *
701  * On %false return, the caller can assume full ownership of @p.
702  */
703 static bool nldsq_cursor_lost_task(struct scx_dsq_list_node *cursor,
704 				   struct rq *rq, struct scx_dispatch_q *dsq,
705 				   struct task_struct *p)
706 {
707 	lockdep_assert_rq_held(rq);
708 	lockdep_assert_held(&dsq->lock);
709 
710 	/*
711 	 * @p could have already left $src_dsq, got re-enqueud, or be in the
712 	 * process of being consumed by someone else.
713 	 */
714 	if (unlikely(p->scx.dsq != dsq ||
715 		     u32_before(cursor->priv, p->scx.dsq_seq) ||
716 		     p->scx.holding_cpu >= 0))
717 		return true;
718 
719 	/* if @p has stayed on @dsq, its rq couldn't have changed */
720 	if (WARN_ON_ONCE(rq != task_rq(p)))
721 		return true;
722 
723 	return false;
724 }
725 
726 /*
727  * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse]
728  * dispatch order. BPF-visible iterator is opaque and larger to allow future
729  * changes without breaking backward compatibility. Can be used with
730  * bpf_for_each(). See bpf_iter_scx_dsq_*().
731  */
732 struct bpf_iter_scx_dsq_kern {
733 	struct scx_dsq_list_node	cursor;
734 	struct scx_dispatch_q		*dsq;
735 	u64				slice;
736 	u64				vtime;
737 } __attribute__((aligned(8)));
738 
739 struct bpf_iter_scx_dsq {
740 	u64				__opaque[6];
741 } __attribute__((aligned(8)));
742 
743 
744 u32 scx_get_task_state(const struct task_struct *p)
745 {
746 	return p->scx.flags & SCX_TASK_STATE_MASK;
747 }
748 
749 void scx_set_task_state(struct task_struct *p, u32 state)
750 {
751 	u32 prev_state = scx_get_task_state(p);
752 	bool warn = false;
753 
754 	switch (state) {
755 	case SCX_TASK_NONE:
756 		warn = prev_state == SCX_TASK_DEAD;
757 		break;
758 	case SCX_TASK_INIT_BEGIN:
759 		warn = prev_state != SCX_TASK_NONE;
760 		break;
761 	case SCX_TASK_INIT:
762 		warn = prev_state != SCX_TASK_INIT_BEGIN;
763 		p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
764 		break;
765 	case SCX_TASK_READY:
766 		warn = !(prev_state == SCX_TASK_INIT ||
767 			 prev_state == SCX_TASK_ENABLED);
768 		break;
769 	case SCX_TASK_ENABLED:
770 		warn = prev_state != SCX_TASK_READY;
771 		break;
772 	case SCX_TASK_DEAD:
773 		warn = !(prev_state == SCX_TASK_NONE ||
774 			 prev_state == SCX_TASK_INIT_BEGIN);
775 		break;
776 	default:
777 		WARN_ONCE(1, "sched_ext: Invalid task state %d -> %d for %s[%d]",
778 			  prev_state, state, p->comm, p->pid);
779 		return;
780 	}
781 
782 	WARN_ONCE(warn, "sched_ext: Invalid task state transition 0x%x -> 0x%x for %s[%d]",
783 		  prev_state, state, p->comm, p->pid);
784 
785 	p->scx.flags &= ~SCX_TASK_STATE_MASK;
786 	p->scx.flags |= state;
787 }
788 
789 /**
790  * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration
791  * @iter: iterator to init
792  * @cgrp: Optional root of cgroup subhierarchy to iterate
793  *
794  * Initialize @iter. Once initialized, @iter must eventually be stopped with
795  * scx_task_iter_stop().
796  *
797  * If @cgrp is %NULL, scx_tasks is used for iteration and this function returns
798  * with scx_tasks_lock held and @iter->cursor inserted into scx_tasks.
799  *
800  * If @cgrp is not %NULL, @cgrp and its descendants' tasks are walked using
801  * @iter->css_iter. The caller must be holding cgroup_lock() to prevent cgroup
802  * task migrations.
803  *
804  * The two modes of iterations are largely independent and it's likely that
805  * scx_tasks can be removed in favor of always using cgroup iteration if
806  * CONFIG_SCHED_CLASS_EXT depends on CONFIG_CGROUPS.
807  *
808  * scx_tasks_lock and the rq lock may be released using scx_task_iter_unlock()
809  * between this and the first next() call or between any two next() calls. If
810  * the locks are released between two next() calls, the caller is responsible
811  * for ensuring that the task being iterated remains accessible either through
812  * RCU read lock or obtaining a reference count.
813  *
814  * All tasks which existed when the iteration started are guaranteed to be
815  * visited as long as they are not dead.
816  */
817 void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp)
818 {
819 	memset(iter, 0, sizeof(*iter));
820 
821 #ifdef CONFIG_EXT_SUB_SCHED
822 	if (cgrp) {
823 		lockdep_assert_held(&cgroup_mutex);
824 		iter->cgrp = cgrp;
825 		iter->css_pos = css_next_descendant_pre(NULL, &iter->cgrp->self);
826 		css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD,
827 				    &iter->css_iter);
828 		return;
829 	}
830 #endif
831 	raw_spin_lock_irq(&scx_tasks_lock);
832 
833 	iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
834 	list_add(&iter->cursor.tasks_node, &scx_tasks);
835 	iter->list_locked = true;
836 }
837 
838 static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter)
839 {
840 	if (iter->locked_task) {
841 		__balance_callbacks(iter->rq, &iter->rf);
842 		task_rq_unlock(iter->rq, iter->locked_task, &iter->rf);
843 		iter->locked_task = NULL;
844 	}
845 }
846 
847 /**
848  * scx_task_iter_unlock - Unlock rq and scx_tasks_lock held by a task iterator
849  * @iter: iterator to unlock
850  *
851  * If @iter is in the middle of a locked iteration, it may be locking the rq of
852  * the task currently being visited in addition to scx_tasks_lock. Unlock both.
853  * This function can be safely called anytime during an iteration. The next
854  * iterator operation will automatically restore the necessary locking.
855  */
856 void scx_task_iter_unlock(struct scx_task_iter *iter)
857 {
858 	__scx_task_iter_rq_unlock(iter);
859 	if (iter->list_locked) {
860 		iter->list_locked = false;
861 		raw_spin_unlock_irq(&scx_tasks_lock);
862 	}
863 }
864 
865 static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter)
866 {
867 	if (!iter->list_locked) {
868 		raw_spin_lock_irq(&scx_tasks_lock);
869 		iter->list_locked = true;
870 	}
871 }
872 
873 /**
874  * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq
875  * @iter: iterator to relock
876  * @p: task whose rq to lock, or %NULL for scx_tasks_lock only
877  *
878  * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once
879  * re-acquired, both locks are managed by the iterator from here on.
880  */
881 static void scx_task_iter_relock(struct scx_task_iter *iter,
882 				 struct task_struct *p)
883 {
884 	__scx_task_iter_maybe_relock(iter);
885 	if (p) {
886 		iter->rq = task_rq_lock(p, &iter->rf);
887 		iter->locked_task = p;
888 	}
889 }
890 
891 /**
892  * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock
893  * @iter: iterator to exit
894  *
895  * Exit a previously initialized @iter. Must be called with scx_tasks_lock held
896  * which is released on return. If the iterator holds a task's rq lock, that rq
897  * lock is also released. See scx_task_iter_start() for details.
898  */
899 void scx_task_iter_stop(struct scx_task_iter *iter)
900 {
901 #ifdef CONFIG_EXT_SUB_SCHED
902 	if (iter->cgrp) {
903 		if (iter->css_pos)
904 			css_task_iter_end(&iter->css_iter);
905 		__scx_task_iter_rq_unlock(iter);
906 		return;
907 	}
908 #endif
909 	__scx_task_iter_maybe_relock(iter);
910 	list_del_init(&iter->cursor.tasks_node);
911 	scx_task_iter_unlock(iter);
912 }
913 
914 /**
915  * scx_task_iter_next - Next task
916  * @iter: iterator to walk
917  *
918  * Visit the next task. See scx_task_iter_start() for details. Locks are dropped
919  * and re-acquired every %SCX_TASK_ITER_BATCH iterations to avoid causing stalls
920  * by holding scx_tasks_lock for too long.
921  */
922 static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)
923 {
924 	struct list_head *cursor = &iter->cursor.tasks_node;
925 	struct sched_ext_entity *pos;
926 
927 	if (!(++iter->cnt % SCX_TASK_ITER_BATCH)) {
928 		scx_task_iter_unlock(iter);
929 		cond_resched();
930 	}
931 
932 #ifdef CONFIG_EXT_SUB_SCHED
933 	if (iter->cgrp) {
934 		while (iter->css_pos) {
935 			struct task_struct *p;
936 
937 			p = css_task_iter_next(&iter->css_iter);
938 			if (p)
939 				return p;
940 
941 			css_task_iter_end(&iter->css_iter);
942 			iter->css_pos = css_next_descendant_pre(iter->css_pos,
943 								&iter->cgrp->self);
944 			if (iter->css_pos)
945 				css_task_iter_start(iter->css_pos, CSS_TASK_ITER_WITH_DEAD,
946 						    &iter->css_iter);
947 		}
948 		return NULL;
949 	}
950 #endif
951 	__scx_task_iter_maybe_relock(iter);
952 
953 	list_for_each_entry(pos, cursor, tasks_node) {
954 		if (&pos->tasks_node == &scx_tasks)
955 			return NULL;
956 		if (!(pos->flags & SCX_TASK_CURSOR)) {
957 			list_move(cursor, &pos->tasks_node);
958 			return container_of(pos, struct task_struct, scx);
959 		}
960 	}
961 
962 	/* can't happen, should always terminate at scx_tasks above */
963 	BUG();
964 }
965 
966 /**
967  * scx_task_iter_next_locked - Next non-idle task with its rq locked
968  * @iter: iterator to walk
969  *
970  * Visit the non-idle task with its rq lock held. Allows callers to specify
971  * whether they would like to filter out dead tasks. See scx_task_iter_start()
972  * for details.
973  */
974 struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
975 {
976 	struct task_struct *p;
977 
978 	__scx_task_iter_rq_unlock(iter);
979 
980 	while ((p = scx_task_iter_next(iter))) {
981 		/*
982 		 * scx_task_iter is used to prepare and move tasks into SCX
983 		 * while loading the BPF scheduler and vice-versa while
984 		 * unloading. The init_tasks ("swappers") should be excluded
985 		 * from the iteration because:
986 		 *
987 		 * - It's unsafe to use __setscheduler_class() on an init_task
988 		 *   to determine the sched_class to use as it won't preserve
989 		 *   its idle_sched_class.
990 		 *
991 		 * - ops.init/exit_task() can easily be confused if called with
992 		 *   init_tasks as they, e.g., share PID 0.
993 		 *
994 		 * As init_tasks are never scheduled through SCX, they can be
995 		 * skipped safely. Note that is_idle_task() which tests %PF_IDLE
996 		 * doesn't work here:
997 		 *
998 		 * - %PF_IDLE may not be set for an init_task whose CPU hasn't
999 		 *   yet been onlined.
1000 		 *
1001 		 * - %PF_IDLE can be set on tasks that are not init_tasks. See
1002 		 *   play_idle_precise() used by CONFIG_IDLE_INJECT.
1003 		 *
1004 		 * Test for idle_sched_class as only init_tasks are on it.
1005 		 */
1006 		if (p->sched_class == &idle_sched_class)
1007 			continue;
1008 
1009 		iter->rq = task_rq_lock(p, &iter->rf);
1010 		iter->locked_task = p;
1011 
1012 		/*
1013 		 * cgroup_task_dead() removes the dead tasks from cset->tasks
1014 		 * after sched_ext_dead() and cgroup iteration may see tasks
1015 		 * which already finished sched_ext_dead(). %SCX_TASK_DEAD is
1016 		 * set by sched_ext_dead() under @p's rq lock. Test it to
1017 		 * avoid visiting tasks which are already dead from SCX POV.
1018 		 */
1019 		if (scx_get_task_state(p) == SCX_TASK_DEAD) {
1020 			__scx_task_iter_rq_unlock(iter);
1021 			continue;
1022 		}
1023 
1024 		return p;
1025 	}
1026 	return NULL;
1027 }
1028 
1029 /**
1030  * scx_dump_event - Dump an event 'kind' in 'events' to 's'
1031  * @s: output seq_buf
1032  * @events: event stats
1033  * @kind: a kind of event to dump
1034  */
1035 #define scx_dump_event(s, events, kind) do {					\
1036 	scx_dump_line(&(s), "%40s: %16lld", #kind, (events)->kind);		\
1037 } while (0)
1038 
1039 
1040 static void scx_read_events(struct scx_sched *sch,
1041 			    struct scx_event_stats *events);
1042 
1043 static enum scx_enable_state scx_enable_state(void)
1044 {
1045 	return atomic_read(&scx_enable_state_var);
1046 }
1047 
1048 static enum scx_enable_state scx_set_enable_state(enum scx_enable_state to)
1049 {
1050 	return atomic_xchg(&scx_enable_state_var, to);
1051 }
1052 
1053 static bool scx_tryset_enable_state(enum scx_enable_state to,
1054 				    enum scx_enable_state from)
1055 {
1056 	int from_v = from;
1057 
1058 	return atomic_try_cmpxchg(&scx_enable_state_var, &from_v, to);
1059 }
1060 
1061 /**
1062  * wait_ops_state - Busy-wait the specified ops state to end
1063  * @p: target task
1064  * @opss: state to wait the end of
1065  *
1066  * Busy-wait for @p to transition out of @opss. This can only be used when the
1067  * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also
1068  * has load_acquire semantics to ensure that the caller can see the updates made
1069  * in the enqueueing and dispatching paths.
1070  */
1071 static void wait_ops_state(struct task_struct *p, unsigned long opss)
1072 {
1073 	do {
1074 		cpu_relax();
1075 	} while (atomic_long_read_acquire(&p->scx.ops_state) == opss);
1076 }
1077 
1078 static inline bool __cpu_valid(s32 cpu)
1079 {
1080 	return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu));
1081 }
1082 
1083 /**
1084  * scx_cpu_valid - Verify a cpu number, to be used on ops input args
1085  * @sch: scx_sched to abort on error
1086  * @cpu: cpu number which came from a BPF ops
1087  * @where: extra information reported on error
1088  *
1089  * @cpu is a cpu number which came from the BPF scheduler and can be any value.
1090  * Verify that it is in range and one of the possible cpus. If invalid, trigger
1091  * an ops error.
1092  */
1093 bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where)
1094 {
1095 	if (__cpu_valid(cpu)) {
1096 		return true;
1097 	} else {
1098 		scx_error(sch, "invalid CPU %d%s%s", cpu, where ? " " : "", where ?: "");
1099 		return false;
1100 	}
1101 }
1102 
1103 static void deferred_bal_cb_workfn(struct rq *rq)
1104 {
1105 	run_deferred(rq);
1106 }
1107 
1108 static void deferred_irq_workfn(struct irq_work *irq_work)
1109 {
1110 	struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work);
1111 
1112 	raw_spin_rq_lock(rq);
1113 	run_deferred(rq);
1114 	scx_rq_lock_drop(rq);
1115 	raw_spin_rq_unlock(rq);
1116 }
1117 
1118 /**
1119  * schedule_deferred - Schedule execution of deferred actions on an rq
1120  * @rq: target rq
1121  *
1122  * Schedule execution of deferred actions on @rq. Deferred actions are executed
1123  * with @rq locked but unpinned, and thus can unlock @rq to e.g. migrate tasks
1124  * to other rqs.
1125  */
1126 static void schedule_deferred(struct rq *rq)
1127 {
1128 	/*
1129 	 * This is the fallback when schedule_deferred_locked() can't use
1130 	 * the cheaper balance callback or wakeup hook paths (the target
1131 	 * CPU is not in dispatch or wakeup). Currently, this is primarily
1132 	 * hit by reenqueue operations targeting a remote CPU.
1133 	 *
1134 	 * Queue on the target CPU. The deferred work can run from any CPU
1135 	 * correctly - the _locked() path already processes remote rqs from
1136 	 * the calling CPU - but targeting the owning CPU allows IPI delivery
1137 	 * without waiting for the calling CPU to re-enable IRQs and is
1138 	 * cheaper as the reenqueue runs locally.
1139 	 */
1140 	irq_work_queue_on(&rq->scx.deferred_irq_work, cpu_of(rq));
1141 }
1142 
1143 /**
1144  * schedule_deferred_locked - Schedule execution of deferred actions on an rq
1145  * @rq: target rq
1146  *
1147  * Schedule execution of deferred actions on @rq. Equivalent to
1148  * schedule_deferred() but requires @rq to be locked and can be more efficient.
1149  */
1150 static void schedule_deferred_locked(struct rq *rq)
1151 {
1152 	lockdep_assert_rq_held(rq);
1153 
1154 	/*
1155 	 * If in the middle of waking up a task, task_woken_scx() will be called
1156 	 * afterwards which will then run the deferred actions, no need to
1157 	 * schedule anything.
1158 	 */
1159 	if (rq->scx.flags & SCX_RQ_IN_WAKEUP)
1160 		return;
1161 
1162 	/* Don't do anything if there already is a deferred operation. */
1163 	if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)
1164 		return;
1165 
1166 	/*
1167 	 * If in dispatch without proxy execution, the balance callbacks will
1168 	 * be called before rq lock is released. Schedule one. With proxy
1169 	 * execution, proxy resolution can zap the callbacks without another
1170 	 * SCX dispatch to requeue them, so use irq_work instead.
1171 	 *
1172 	 * We can't directly insert the callback into the
1173 	 * rq's list: The call can drop its lock and make the pending balance
1174 	 * callback visible to unrelated code paths that call rq_pin_lock().
1175 	 *
1176 	 * Just let dispatch_one() know that it must do it itself.
1177 	 */
1178 	if ((rq->scx.flags & SCX_RQ_IN_DISPATCH) && !sched_proxy_exec()) {
1179 		rq->scx.flags |= SCX_RQ_BAL_CB_PENDING;
1180 		return;
1181 	}
1182 
1183 	/*
1184 	 * No scheduler hooks available. Use the generic irq_work path. The
1185 	 * above WAKEUP and DISPATCH paths should cover most of the cases and
1186 	 * the time to IRQ re-enable shouldn't be long.
1187 	 */
1188 	schedule_deferred(rq);
1189 }
1190 
1191 static void resched_irq_workfn(struct irq_work *irq_work)
1192 {
1193 	struct rq *rq = container_of(irq_work, struct rq, scx.resched_irq_work);
1194 
1195 	guard(rq_lock)(rq);
1196 	/* a higher class task would only be picked again, see kick_one_cpu() */
1197 	if (!sched_class_above(rq->donor->sched_class, &ext_sched_class))
1198 		resched_curr(rq);
1199 	scx_rq_lock_drop(rq);
1200 }
1201 
1202 /**
1203  * scx_resched_curr_deferred - Reschedule @rq's cpu from hard irq work on it
1204  * @rq: rq to reschedule
1205  *
1206  * A kick, like a direct resched, only guarantees that a scheduling event on the
1207  * cpu completes after it, and that can be the one in progress. Work that a pick
1208  * processes only at the start of its dispatch, such as the ecaps syncs, can be
1209  * queued after that point and miss the pick. A pick on @rq's cpu runs with IRQs
1210  * off, so the irq work runs after it, and the scheduling event its resched
1211  * triggers starts after this call. A sibling's core scheduling pick for @rq
1212  * isn't covered. A cpu running a task of a higher class isn't rescheduled, as
1213  * its pick would return that task again. @rq's cpu must stay online until the
1214  * irq work is queued.
1215  */
1216 void scx_resched_curr_deferred(struct rq *rq)
1217 {
1218 	irq_work_queue_on(&rq->scx.resched_irq_work, cpu_of(rq));
1219 }
1220 
1221 #ifdef CONFIG_NO_HZ_FULL
1222 static void scx_proxy_tick_bal_cb(struct rq *rq)
1223 {
1224 	sched_update_tick_dependency(rq);
1225 }
1226 
1227 static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next)
1228 {
1229 	bool proxy = next != rq->donor;
1230 	bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK;
1231 
1232 	if (likely(proxy == was_proxy))
1233 		return;
1234 
1235 	if (proxy) {
1236 		/* Keep proxy execution tick-driven for now. */
1237 		rq->scx.flags |= SCX_RQ_PROXY_TICK;
1238 		tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
1239 	} else {
1240 		rq->scx.flags &= ~SCX_RQ_PROXY_TICK;
1241 		/*
1242 		 * The selected donor is already visible, but rq->curr still
1243 		 * identifies the outgoing execution context. Reevaluate after
1244 		 * context_switch() updates rq->curr so sched_can_stop_tick() sees
1245 		 * the complete selection.
1246 		 */
1247 		queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb,
1248 				       scx_proxy_tick_bal_cb);
1249 	}
1250 }
1251 #endif
1252 
1253 /*
1254  * Complete sched_ext bookkeeping after proxy resolution and retry tasks which
1255  * couldn't be reenqueued by an earlier reject DSQ drain.
1256  */
1257 void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next)
1258 {
1259 	lockdep_assert_rq_held(rq);
1260 
1261 #ifdef CONFIG_NO_HZ_FULL
1262 	if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq)))
1263 		scx_proxy_update_tick(rq, next);
1264 #endif
1265 
1266 	/*
1267 	 * A higher priority class may wake up while @rq is transiently idle
1268 	 * for proxy resolution. wakeup_preempt_scx() isn't invoked in that
1269 	 * case, so reenqueue IMMED donors parked on the local DSQ here.
1270 	 */
1271 	if (rq->scx.nr_immed && sched_class_above(rq->next_class, &ext_sched_class))
1272 		scx_schedule_reenq_local(rq, 0);
1273 
1274 	if (rq->scx.flags & SCX_RQ_PROXY_RETRY) {
1275 		rq->scx.flags &= ~SCX_RQ_PROXY_RETRY;
1276 		schedule_deferred_locked(rq);
1277 	}
1278 }
1279 
1280 void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
1281 			u64 reenq_flags, struct rq *locked_rq)
1282 {
1283 	struct rq *rq;
1284 
1285 	/*
1286 	 * Allowing reenqueues doesn't make sense while bypassing. This also
1287 	 * blocks from new reenqueues to be scheduled on dead scheds.
1288 	 */
1289 	if (unlikely(READ_ONCE(sch->bypass_depth)))
1290 		return;
1291 
1292 	if (dsq->id == SCX_DSQ_LOCAL) {
1293 		rq = container_of(dsq, struct rq, scx.local_dsq);
1294 
1295 		/*
1296 		 * An inactive cpu's local DSQ holds only tasks that are bound
1297 		 * to it or are being moved off, so a reenq of it is pointless.
1298 		 * Once balance push is installed, queue_balance_callback() also
1299 		 * drops the callback that would run a reenq requested from
1300 		 * dispatch, and irq_work_queue_on() can't reach an offline cpu.
1301 		 * The reenq's node would then stay on this rq's list after
1302 		 * @sch's pcpu is freed.
1303 		 */
1304 		if (unlikely(!cpu_active(cpu_of(rq))))
1305 			return;
1306 
1307 		/*
1308 		 * A sub-sched lacking baseline access on the target cid has no
1309 		 * business triggering IPIs. The lockless test is fine: slipping
1310 		 * through right after a revoke is harmless and a wrong denial
1311 		 * can't happen - if the caller has seen its ownership, so does
1312 		 * this test.
1313 		 */
1314 		if (unlikely(scx_missing_caps(sch, cpu_of(rq), SCX_CAP_BASE))) {
1315 			__scx_add_event(sch, SCX_EV_SUB_REENQ_DENIED, 1);
1316 			return;
1317 		}
1318 
1319 		struct scx_sched_pcpu *sch_pcpu = per_cpu_ptr(sch->pcpu, cpu_of(rq));
1320 		struct scx_deferred_reenq_local *drl = &sch_pcpu->deferred_reenq_local;
1321 
1322 		/*
1323 		 * Pairs with smp_mb() in process_deferred_reenq_locals() and
1324 		 * guarantees that there is a reenq_local() afterwards.
1325 		 */
1326 		smp_mb();
1327 
1328 		if (list_empty(&drl->node) ||
1329 		    (READ_ONCE(drl->flags) & reenq_flags) != reenq_flags) {
1330 
1331 			guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
1332 
1333 			if (list_empty(&drl->node))
1334 				list_move_tail(&drl->node, &rq->scx.deferred_reenq_locals);
1335 			WRITE_ONCE(drl->flags, drl->flags | reenq_flags);
1336 		}
1337 	} else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) {
1338 		rq = this_rq();
1339 
1340 		struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq));
1341 		struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user;
1342 
1343 		/*
1344 		 * Pairs with smp_mb() in process_deferred_reenq_users() and
1345 		 * guarantees that there is a reenq_user() afterwards.
1346 		 */
1347 		smp_mb();
1348 
1349 		if (list_empty(&dru->node) ||
1350 		    (READ_ONCE(dru->flags) & reenq_flags) != reenq_flags) {
1351 
1352 			guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
1353 
1354 			/* @dsq is being destroyed, see free_dsq_irq_workfn() */
1355 			if (unlikely(READ_ONCE(dsq->id) == SCX_DSQ_INVALID))
1356 				return;
1357 
1358 			if (list_empty(&dru->node))
1359 				list_move_tail(&dru->node, &rq->scx.deferred_reenq_users);
1360 			WRITE_ONCE(dru->flags, dru->flags | reenq_flags);
1361 		}
1362 	} else {
1363 		/* destroyed since the caller resolved it, nothing to reenq */
1364 		if (READ_ONCE(dsq->id) != SCX_DSQ_INVALID)
1365 			scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id);
1366 		return;
1367 	}
1368 
1369 	if (rq == locked_rq)
1370 		schedule_deferred_locked(rq);
1371 	else
1372 		schedule_deferred(rq);
1373 }
1374 
1375 /*
1376  * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero
1377  * word means no request. Otherwise the fields are:
1378  *
1379  *   63      SCX_SLICE_OOB_PENDING, set on every request
1380  *   62-43   lower bits of issuing scheduler's id
1381  *   42-0    requested slice duration in nsecs
1382  *
1383  * A duration of SCX_SLICE_OOB_DUR_MASK means SCX_SLICE_INF. A finite dur
1384  * saturates at SCX_SLICE_OOB_DUR_MASK - 1. The id is used to detect and ignore
1385  * a request that outlived a task ownership change.
1386  *
1387  * Only the low 20 bits of sch->id are packed, which is enough to make
1388  * collisions practically impossible. A theoretical collision just lets a stale
1389  * request through once.
1390  */
1391 enum scx_slice_oob_consts {
1392 	SCX_SLICE_OOB_DUR_BITS	= 43,
1393 	SCX_SLICE_OOB_ID_BITS	= 64 - SCX_SLICE_OOB_DUR_BITS - 1,
1394 
1395 	SCX_SLICE_OOB_DUR_MASK	= (1LLU << SCX_SLICE_OOB_DUR_BITS) - 1,
1396 	SCX_SLICE_OOB_ID_SHIFT	= SCX_SLICE_OOB_DUR_BITS,
1397 	SCX_SLICE_OOB_ID_MASK	= (1LLU << SCX_SLICE_OOB_ID_BITS) - 1,
1398 	SCX_SLICE_OOB_PENDING	= 1LLU << 63,
1399 };
1400 
1401 /*
1402  * Slice and dsq_vtime write rules
1403  *
1404  * While @p is running, sleeping or queued on an rq-owned DSQ, both fields are
1405  * protected by the rq lock. While running, the rq lock is required because
1406  * update_curr_scx() RMWs the slice and the cap check for slice extension is
1407  * only reliable under the rq lock.
1408  *
1409  * While @p is queued on a user DSQ or on the BPF side, the kernel neither
1410  * consumes nor decides on the fields. Synchronizing the writers is the BPF
1411  * scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a
1412  * concurrent DSQ insertion commit can race each other and whichever lands last
1413  * wins.
1414  *
1415  * A DSQ insert kfunc doesn't update the fields directly. The verdict carries
1416  * the values and apply_slice_vtime() commits them at the insertion.
1417  *
1418  * scx_bpf_task_set_slice() may be called from any context and writes directly
1419  * only if @p's rq lock is already held, otherwise it bounces through
1420  * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration.
1421  *
1422  * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is
1423  * refused. See set_task_slice_keep_oob(). A kernel slice, see
1424  * %SCX_TASK_KERNEL_SLICE, can be shortened but not extended.
1425  *
1426  * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so
1427  * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct
1428  * BPF stores from non-cid-form schedulers are outside these rules.
1429  */
1430 
1431 /* clear a pending slice request */
1432 static void clear_task_slice_oob(struct task_struct *p)
1433 {
1434 	if (unlikely(atomic64_read(&p->scx.slice_oob)))
1435 		atomic64_set(&p->scx.slice_oob, 0);
1436 }
1437 
1438 /**
1439  * dsq_insert_head - FIFO head insertion honoring %SCX_TASK_PROTECTED
1440  * @dsq: DSQ to insert into
1441  * @p: task being inserted
1442  *
1443  * A HEAD insert should land behind any leading protected tasks. Return %true
1444  * indicates whether @p became the first entry.
1445  */
1446 static bool dsq_insert_head(struct scx_dispatch_q *dsq, struct task_struct *p)
1447 {
1448 	struct list_head *pos = &dsq->list;
1449 	struct scx_dsq_list_node *node;
1450 
1451 	/*
1452 	 * Only rq-owned DSQs can hold protected tasks and the associated rq
1453 	 * lock keeps their flags stable.
1454 	 */
1455 	if (!dsq_is_rq_owned(dsq)) {
1456 		list_add(&p->scx.dsq_list.node, &dsq->list);
1457 		return true;
1458 	}
1459 
1460 	list_for_each_entry(node, &dsq->list, node) {
1461 		struct task_struct *q;
1462 
1463 		if (WARN_ON_ONCE(node->flags & SCX_DSQ_LNODE_ITER_CURSOR))
1464 			continue;
1465 
1466 		q = container_of(node, struct task_struct, scx.dsq_list);
1467 		if (!(q->scx.flags & SCX_TASK_PROTECTED))
1468 			break;
1469 
1470 		pos = &node->node;
1471 	}
1472 
1473 	list_add(&p->scx.dsq_list.node, pos);
1474 
1475 	return pos == &dsq->list;
1476 }
1477 
1478 /**
1479  * set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request
1480  * @p: task of interest
1481  * @slice: slice to set
1482  *
1483  * While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice.
1484  * Refuse and return %false.
1485  */
1486 static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice)
1487 {
1488 	lockdep_assert_rq_held(task_rq(p));
1489 
1490 	if (unlikely(p->scx.flags & SCX_TASK_PROTECTED))
1491 		return false;
1492 
1493 	p->scx.slice = slice;
1494 	return true;
1495 }
1496 
1497 /* set @p's slice, superseding any pending out-of-band request */
1498 bool scx_set_task_slice(struct task_struct *p, u64 slice)
1499 {
1500 	if (!set_task_slice_keep_oob(p, slice))
1501 		return false;
1502 	clear_task_slice_oob(p);
1503 	return true;
1504 }
1505 
1506 /**
1507  * scx_task_slice_ended - @p's slice is consumed or given up
1508  * @rq: rq @p is on
1509  * @p: task of interest
1510  *
1511  * End what rides on the slice - the protection, the kernel's ownership of the
1512  * slice, and the rescue if @p is being rescued.
1513  *
1514  * A dequeue normally ends the slice too. The exception is a save/restore pair
1515  * on the running task. Attribute changes like renice cycle the task through
1516  * dequeue and enqueue while it keeps executing, so the slice continues. A
1517  * queued task instead loses its DSQ position on any dequeue and the slice ends
1518  * with it.
1519  */
1520 void scx_task_slice_ended(struct rq *rq, struct task_struct *p)
1521 {
1522 	lockdep_assert_rq_held(rq);
1523 
1524 	p->scx.flags &= ~(SCX_TASK_PROTECTED | SCX_TASK_KERNEL_SLICE);
1525 	if (unlikely(p == scx_rescuee(rq)))
1526 		scx_rescue_end(rq);
1527 }
1528 
1529 /* request @p's slice to be set to @slice, see the write rules above */
1530 static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice)
1531 {
1532 	u64 dur;
1533 
1534 	if (slice == SCX_SLICE_INF) {
1535 		dur = SCX_SLICE_OOB_DUR_MASK;
1536 	} else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) {
1537 		dur = SCX_SLICE_OOB_DUR_MASK - 1;
1538 		scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1);
1539 	} else {
1540 		dur = slice;
1541 	}
1542 
1543 	atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING |
1544 		     ((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) | dur);
1545 }
1546 
1547 /*
1548  * Apply a pending out-of-band slice request under @rq's lock. A request whose
1549  * packed id no longer matches @p's current owner is dropped. An extension needs
1550  * baseline cpu access on @p's cid and is refused on a kernel slice. Shortening
1551  * is always allowed. A protected slice refuses both. %SCX_EV_SLICE_DENIED
1552  * counts the denials. See the write rules above.
1553  */
1554 static void apply_task_slice_oob(struct rq *rq, struct task_struct *p)
1555 {
1556 	u64 oob, dur, slice;
1557 
1558 	lockdep_assert_rq_held(rq);
1559 
1560 	if (likely(!atomic64_read(&p->scx.slice_oob)))
1561 		return;
1562 
1563 	oob = atomic64_xchg(&p->scx.slice_oob, 0);
1564 	if (unlikely(!oob))
1565 		return;
1566 
1567 	/* the issuing scheduler no longer owns @p, drop the request */
1568 	if (unlikely(((oob >> SCX_SLICE_OOB_ID_SHIFT) & SCX_SLICE_OOB_ID_MASK) !=
1569 		     (scx_task_sched(p)->id & SCX_SLICE_OOB_ID_MASK)))
1570 		return;
1571 
1572 	dur = oob & SCX_SLICE_OOB_DUR_MASK;
1573 	slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur;
1574 
1575 	if (slice > p->scx.slice &&
1576 	    unlikely(scx_task_has_kernel_slice(p) ||
1577 		     scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) {
1578 		__scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1);
1579 		return;
1580 	}
1581 
1582 	if (unlikely(!set_task_slice_keep_oob(p, slice)))
1583 		__scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1);
1584 }
1585 
1586 /*
1587  * A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them
1588  * and they are committed here, at the insertion. A zero @slice keeps the
1589  * current value, floored at 1 so the task isn't treated as expired.
1590  */
1591 static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags)
1592 {
1593 	if (slice) {
1594 		p->scx.slice = slice;
1595 		/*
1596 		 * An explicit slice supersedes a pending oob request. A carried
1597 		 * default refill is not an explicit request and must keep it.
1598 		 */
1599 		if (!(enq_flags & SCX_ENQ_SLICE_DFL))
1600 			clear_task_slice_oob(p);
1601 	} else if (!p->scx.slice) {
1602 		p->scx.slice = 1;
1603 	}
1604 
1605 	if (enq_flags & SCX_ENQ_DSQ_PRIQ)
1606 		p->scx.dsq_vtime = vtime;
1607 }
1608 
1609 /*
1610  * refill_task_slice_dfl() supplies a slice when the scheduler left none and
1611  * expire_task_slice() takes it away to force the task off the cpu. Neither
1612  * touches a kernel slice, see %SCX_TASK_KERNEL_SLICE.
1613  */
1614 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)
1615 {
1616 	if (scx_task_has_kernel_slice(p))
1617 		return;
1618 
1619 	/*
1620 	 * A default refill is not an explicit request, so it must not drop a
1621 	 * pending out-of-band one, which is applied when @p next runs.
1622 	 */
1623 	set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl));
1624 	__scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
1625 }
1626 
1627 static void expire_task_slice(struct task_struct *p)
1628 {
1629 	/*
1630 	 * Unlike the refill above, an expiry supersedes a pending out-of-band
1631 	 * request. A kept request would be applied at the next
1632 	 * update_curr_scx() and undo the expiry.
1633 	 */
1634 	if (!scx_task_has_kernel_slice(p))
1635 		scx_set_task_slice(p, 0);
1636 }
1637 
1638 static void update_curr_scx(struct rq *rq)
1639 {
1640 	struct task_struct *donor;
1641 	s64 delta_exec;
1642 
1643 	/*
1644 	 * update_curr_scx() is selected through rq->donor->sched_class, not
1645 	 * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT
1646 	 * owner executes for a FAIR donor, FAIR's update_curr() runs instead.
1647 	 */
1648 	donor = rq->donor;
1649 
1650 	/* apply even on 0 delta_exec, callers may still act on the slice */
1651 	apply_task_slice_oob(rq, donor);
1652 
1653 	delta_exec = update_curr_common(rq);
1654 	if (unlikely(delta_exec <= 0))
1655 		return;
1656 
1657 	if (donor->scx.slice != SCX_SLICE_INF)
1658 		donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec);
1659 
1660 	if (unlikely(donor == scx_rescuee(rq)))
1661 		scx_rescue_charge(rq, delta_exec);
1662 
1663 	dl_server_update(&rq->ext_server, delta_exec);
1664 }
1665 
1666 static bool scx_dsq_priq_less(struct rb_node *node_a,
1667 			      const struct rb_node *node_b)
1668 {
1669 	const struct task_struct *a =
1670 		container_of(node_a, struct task_struct, scx.dsq_priq);
1671 	const struct task_struct *b =
1672 		container_of(node_b, struct task_struct, scx.dsq_priq);
1673 
1674 	return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime);
1675 }
1676 
1677 static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags)
1678 {
1679 	/* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */
1680 	WRITE_ONCE(dsq->nr, dsq->nr + 1);
1681 
1682 	/*
1683 	 * Once @p reaches a local DSQ, it can only leave it by being dispatched
1684 	 * to the CPU or dequeued. In both cases, the only way @p can go back to
1685 	 * the BPF sched is through enqueueing. If being inserted into a local
1686 	 * DSQ with IMMED, persist the state until the next enqueueing event in
1687 	 * scx_do_enqueue_task() so that we can maintain IMMED protection
1688 	 * through e.g. SAVE/RESTORE cycles and slice extensions.
1689 	 */
1690 	if (enq_flags & SCX_ENQ_IMMED) {
1691 		if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
1692 			WARN_ON_ONCE(!(enq_flags & SCX_ENQ_GDSQ_FALLBACK));
1693 			return;
1694 		}
1695 		p->scx.flags |= SCX_TASK_IMMED;
1696 	}
1697 
1698 	if (p->scx.flags & SCX_TASK_IMMED) {
1699 		struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1700 
1701 		if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
1702 			return;
1703 
1704 		rq->scx.nr_immed++;
1705 
1706 		/*
1707 		 * If @rq already had other tasks or the current task is not
1708 		 * done yet, @p can't go on the CPU immediately. Re-enqueue.
1709 		 */
1710 		if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags)))
1711 			scx_schedule_reenq_local(rq, 0);
1712 	}
1713 }
1714 
1715 static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p)
1716 {
1717 	/* see dsq_inc_nr() */
1718 	WRITE_ONCE(dsq->nr, dsq->nr - 1);
1719 
1720 	if (p->scx.flags & SCX_TASK_IMMED) {
1721 		struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1722 
1723 		if (WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL) ||
1724 		    WARN_ON_ONCE(rq->scx.nr_immed <= 0))
1725 			return;
1726 
1727 		rq->scx.nr_immed--;
1728 	}
1729 }
1730 
1731 /*
1732  * Return true if @p is moving due to an internal SCX migration, false
1733  * otherwise.
1734  */
1735 static inline bool task_scx_migrating(struct task_struct *p)
1736 {
1737 	/*
1738 	 * We only need to check sticky_cpu: it is set to the destination
1739 	 * CPU in move_remote_task_to_local_dsq() before deactivate_task()
1740 	 * and cleared when the task is enqueued on the destination, so it
1741 	 * is only non-negative during an internal SCX migration.
1742 	 */
1743 	return p->scx.sticky_cpu >= 0;
1744 }
1745 
1746 /* Must be called under the lock serializing @p's custody transfers. */
1747 static bool task_leave_custody(struct task_struct *p)
1748 {
1749 	if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p))
1750 		return false;
1751 
1752 	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
1753 	return true;
1754 }
1755 
1756 /*
1757  * A task with an infinite slice may be running with its tick stopped. Lazy
1758  * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee
1759  * that the lazy request is promoted by a real scheduler tick. Set the lazy
1760  * request first so that the dependency update's IPI can also serve it on return
1761  * to user space.
1762  */
1763 static void scx_resched_curr_lazy(struct rq *rq)
1764 {
1765 	resched_curr_lazy(rq);
1766 
1767 	if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
1768 		rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
1769 		if (rq->clock_update_flags < RQCF_UPDATED)
1770 			update_rq_clock(rq);
1771 		update_other_load_avgs(rq);
1772 		sched_update_tick_dependency(rq);
1773 	}
1774 }
1775 
1776 static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
1777 			      struct scx_dispatch_q *dsq, struct task_struct *p,
1778 			      u64 enq_flags)
1779 {
1780 	if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
1781 		SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
1782 
1783 	/*
1784 	 * Only local inserts get the wakeup treatment below. Rejects kick a
1785 	 * deferred reenq and rescue parks are paced by the rescue timer. Proxy
1786 	 * rejects which aren't ready when drained request a later retry from
1787 	 * scx_proxy_reenqueue_retry().
1788 	 */
1789 	if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
1790 		if (dsq->id == SCX_DSQ_REJECT)
1791 			schedule_deferred_locked(rq);
1792 		return;
1793 	}
1794 
1795 	/*
1796 	 * Note that @rq's lock may be dropped between this enqueue and @p
1797 	 * actually getting on CPU. This gives higher-class tasks (e.g. RT)
1798 	 * an opportunity to wake up on @rq and prevent @p from running.
1799 	 * Here are some concrete examples:
1800 	 *
1801 	 * Example 1:
1802 	 *
1803 	 * We dispatch two tasks from a single ops.dispatch():
1804 	 * - First, a local task to this CPU's local DSQ;
1805 	 * - Second, a local/remote task to a remote CPU's local DSQ.
1806 	 * We must drop the local rq lock in order to finish the second
1807 	 * dispatch. In that time, an RT task can wake up on the local rq.
1808 	 *
1809 	 * Example 2:
1810 	 *
1811 	 * We dispatch a local/remote task to a remote CPU's local DSQ.
1812 	 * We must drop the remote rq lock before the dispatched task can run,
1813 	 * which gives an RT task an opportunity to wake up on the remote rq.
1814 	 *
1815 	 * Both examples work the same if we replace dispatching with moving
1816 	 * the tasks from a user-created DSQ.
1817 	 *
1818 	 * We must detect these wakeups so that we can re-enqueue IMMED tasks
1819 	 * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this
1820 	 * purpose, but for it to be invoked, we must ensure that we bump
1821 	 * @rq->next_class to &ext_sched_class if it's currently idle.
1822 	 *
1823 	 * wakeup_preempt() does the bumping, and since we only invoke it if
1824 	 * @rq->next_class is below &ext_sched_class, it will also
1825 	 * resched_curr(rq).
1826 	 */
1827 	if (sched_class_above(p->sched_class, rq->next_class))
1828 		wakeup_preempt(rq, p, 0);
1829 
1830 	/*
1831 	 * If @rq is in dispatch, the CPU is already vacant and looking for the
1832 	 * next task to run. No need to preempt or trigger resched after moving
1833 	 * @p into its local DSQ.
1834 	 * Note that the wakeup_preempt() above may have already triggered
1835 	 * a resched if @rq->next_class was idle. It's harmless, since
1836 	 * need_resched is cleared immediately after task pick.
1837 	 */
1838 	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
1839 		return;
1840 
1841 	if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) &&
1842 	    p != rq->donor && rq->donor->sched_class == &ext_sched_class) {
1843 		if (likely(scx_set_task_slice(rq->donor, 0))) {
1844 			if (enq_flags & SCX_ENQ_PREEMPT)
1845 				resched_curr(rq);
1846 			else
1847 				scx_resched_curr_lazy(rq);
1848 		} else {
1849 			__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
1850 		}
1851 	}
1852 }
1853 
1854 static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
1855 				 struct scx_dispatch_q *dsq, struct task_struct *p,
1856 				 u64 slice, u64 vtime, u64 enq_flags)
1857 {
1858 	bool is_rq_owned = dsq_is_rq_owned(dsq);
1859 
1860 	if (dsq->id == SCX_DSQ_LOCAL)
1861 		dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
1862 
1863 	WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
1864 	WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
1865 		     !RB_EMPTY_NODE(&p->scx.dsq_priq));
1866 
1867 	if (!is_rq_owned) {
1868 		raw_spin_lock_nested(&dsq->lock,
1869 			(enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0);
1870 
1871 		if (unlikely(dsq->id == SCX_DSQ_INVALID)) {
1872 			scx_error(sch, "attempting to dispatch to a destroyed dsq");
1873 			/* fall back to the global dsq */
1874 			raw_spin_unlock(&dsq->lock);
1875 			dsq = find_global_dsq(sch, task_cpu(p));
1876 			raw_spin_lock(&dsq->lock);
1877 		}
1878 	}
1879 
1880 	if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) &&
1881 		     (enq_flags & SCX_ENQ_DSQ_PRIQ))) {
1882 		/*
1883 		 * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from
1884 		 * their FIFO queues. To avoid confusion and accidentally
1885 		 * starving vtime-dispatched tasks by FIFO-dispatched tasks, we
1886 		 * disallow any internal DSQ from doing vtime ordering of
1887 		 * tasks.
1888 		 */
1889 		scx_error(sch, "cannot use vtime ordering for built-in DSQs");
1890 		enq_flags &= ~SCX_ENQ_DSQ_PRIQ;
1891 	}
1892 
1893 	/*
1894 	 * @dsq is locked and @enq_flags is sanitized. Commit the carried slice
1895 	 * and vtime before the PRIQ insertion below reads the new dsq_vtime.
1896 	 */
1897 	if (enq_flags & SCX_ENQ_APPLY_SLICE)
1898 		apply_slice_vtime(p, slice, vtime, enq_flags);
1899 
1900 	if (enq_flags & SCX_ENQ_DSQ_PRIQ) {
1901 		struct rb_node *rbp;
1902 
1903 		/*
1904 		 * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are
1905 		 * linked to both the rbtree and list on PRIQs, this can only be
1906 		 * tested easily when adding the first task.
1907 		 */
1908 		if (unlikely(RB_EMPTY_ROOT(&dsq->priq) &&
1909 			     nldsq_next_task(dsq, NULL, false)))
1910 			scx_error(sch, "DSQ ID 0x%016llx already had FIFO-enqueued tasks",
1911 				  dsq->id);
1912 
1913 		p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ;
1914 		rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less);
1915 
1916 		/*
1917 		 * Find the previous task and insert after it on the list so
1918 		 * that @dsq->list is vtime ordered.
1919 		 */
1920 		rbp = rb_prev(&p->scx.dsq_priq);
1921 		if (rbp) {
1922 			struct task_struct *prev =
1923 				container_of(rbp, struct task_struct,
1924 					     scx.dsq_priq);
1925 			list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node);
1926 			/* first task unchanged - no update needed */
1927 		} else {
1928 			list_add(&p->scx.dsq_list.node, &dsq->list);
1929 			/* not builtin and new task is at head - use fastpath */
1930 			rcu_assign_pointer(dsq->first_task, p);
1931 		}
1932 	} else {
1933 		/* a FIFO DSQ shouldn't be using PRIQ enqueuing */
1934 		if (unlikely(!RB_EMPTY_ROOT(&dsq->priq)))
1935 			scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks",
1936 				  dsq->id);
1937 
1938 		if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) {
1939 			/* new task inserted at head - use fastpath */
1940 			if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1941 				rcu_assign_pointer(dsq->first_task, p);
1942 		} else {
1943 			/*
1944 			 * dsq->list can contain parked BPF iterator cursors, so
1945 			 * list_empty() here isn't a reliable proxy for "no real
1946 			 * task in the DSQ". Test dsq->first_task directly.
1947 			 */
1948 			list_add_tail(&p->scx.dsq_list.node, &dsq->list);
1949 			if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1950 				rcu_assign_pointer(dsq->first_task, p);
1951 		}
1952 	}
1953 
1954 	/* seq records the order tasks are queued, used by BPF DSQ iterator */
1955 	WRITE_ONCE(dsq->seq, dsq->seq + 1);
1956 	p->scx.dsq_seq = dsq->seq;
1957 
1958 	dsq_inc_nr(dsq, p, enq_flags);
1959 	p->scx.dsq = dsq;
1960 
1961 	/*
1962 	 * Update custody and call ops.dequeue() before clearing ops_state:
1963 	 * once ops_state is cleared, waiters in ops_dequeue() can proceed
1964 	 * and dequeue_task_scx() will RMW p->scx.flags. If we clear
1965 	 * ops_state first, both sides would modify p->scx.flags
1966 	 * concurrently in a non-atomic way.
1967 	 */
1968 	if (is_rq_owned) {
1969 		rq_owned_post_enq(sch, rq, dsq, p, enq_flags);
1970 	} else {
1971 		bool call_dequeue = false;
1972 
1973 		/*
1974 		 * Global and bypass DSQs are terminal - the task leaves the
1975 		 * scheduler's custody, so ops.dequeue() fires. It can run
1976 		 * without @p's rq lock (finish_dispatch() passes the dispatch
1977 		 * rq); that's safe because dequeue_task_scx() waits on
1978 		 * SCX_OPSS_DISPATCHING (see the ops_state note above) and so
1979 		 * can't race it. A non-terminal DSQ keeps the task in custody.
1980 		 * The custody transfer happens under @dsq->lock so that later
1981 		 * consumers see the flag clear; the callback runs after
1982 		 * @dsq->lock is dropped because it may lock a DSQ itself.
1983 		 */
1984 		if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS)
1985 			call_dequeue = task_leave_custody(p);
1986 		else
1987 			p->scx.flags |= SCX_TASK_IN_CUSTODY;
1988 
1989 		raw_spin_unlock(&dsq->lock);
1990 
1991 		if (call_dequeue && SCX_HAS_OP(sch, dequeue))
1992 			SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
1993 	}
1994 
1995 	/*
1996 	 * We're transitioning out of QUEUEING or DISPATCHING. store_release to
1997 	 * match waiters' load_acquire.
1998 	 */
1999 	if (enq_flags & SCX_ENQ_CLEAR_OPSS)
2000 		atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
2001 }
2002 
2003 void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq)
2004 {
2005 	WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
2006 
2007 	if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) {
2008 		rb_erase(&p->scx.dsq_priq, &dsq->priq);
2009 		RB_CLEAR_NODE(&p->scx.dsq_priq);
2010 		p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ;
2011 	}
2012 	p->scx.dsq_flags &= ~SCX_TASK_DSQ_REENQ;
2013 
2014 	list_del_init(&p->scx.dsq_list.node);
2015 	dsq_dec_nr(dsq, p);
2016 
2017 	if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN) && rcu_access_pointer(dsq->first_task) == p) {
2018 		struct task_struct *first_task;
2019 
2020 		first_task = nldsq_next_task(dsq, NULL, false);
2021 		rcu_assign_pointer(dsq->first_task, first_task);
2022 	}
2023 }
2024 
2025 void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
2026 {
2027 	struct scx_dispatch_q *dsq = p->scx.dsq;
2028 	bool is_rq_owned = dsq && dsq_is_rq_owned(dsq);
2029 
2030 	lockdep_assert_rq_held(rq);
2031 
2032 	if (!dsq) {
2033 		/*
2034 		 * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals.
2035 		 * Unlinking is all that's needed to cancel.
2036 		 */
2037 		if (unlikely(!list_empty(&p->scx.dsq_list.node)))
2038 			list_del_init(&p->scx.dsq_list.node);
2039 
2040 		/*
2041 		 * When dispatch_to_local_dsq() or remote consumption moves a
2042 		 * task to a local DSQ, the task isn't associated with any DSQ
2043 		 * but @p->scx.holding_cpu may be set. Clearing holding_cpu
2044 		 * tells dispatch_to_local_dsq() that it lost to a dequeue.
2045 		 */
2046 		if (p->scx.holding_cpu >= 0)
2047 			p->scx.holding_cpu = -1;
2048 
2049 		return;
2050 	}
2051 
2052 	if (!is_rq_owned)
2053 		raw_spin_lock(&dsq->lock);
2054 
2055 	/*
2056 	 * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't
2057 	 * change underneath us.
2058 	*/
2059 	if (p->scx.holding_cpu < 0) {
2060 		/* @p must still be on @dsq, dequeue */
2061 		scx_task_unlink_from_dsq(p, dsq);
2062 	} else {
2063 		/*
2064 		 * We're racing against unlink_dsq_and_switch_rq_lock(),
2065 		 * which already removed @p from @dsq and set
2066 		 * @p->scx.holding_cpu. Clear holding_cpu to tell
2067 		 * unlink_dsq_and_switch_rq_lock() that it lost the race.
2068 		 */
2069 		WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
2070 		p->scx.holding_cpu = -1;
2071 	}
2072 	p->scx.dsq = NULL;
2073 
2074 	if (!is_rq_owned)
2075 		raw_spin_unlock(&dsq->lock);
2076 }
2077 
2078 /*
2079  * Abbreviated version of scx_dispatch_dequeue() that can be used when both
2080  * @p's rq and dsq are locked.
2081  */
2082 static void dispatch_dequeue_locked(struct task_struct *p,
2083 				    struct scx_dispatch_q *dsq)
2084 {
2085 	lockdep_assert_rq_held(task_rq(p));
2086 	lockdep_assert_held(&dsq->lock);
2087 
2088 	scx_task_unlink_from_dsq(p, dsq);
2089 	p->scx.dsq = NULL;
2090 }
2091 
2092 static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
2093 						    struct rq *rq, u64 dsq_id,
2094 						    s32 tcpu)
2095 {
2096 	struct scx_dispatch_q *dsq;
2097 
2098 	if (dsq_id == SCX_DSQ_LOCAL)
2099 		return &rq->scx.local_dsq;
2100 
2101 	if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
2102 		s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
2103 
2104 		if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict"))
2105 			return find_global_dsq(sch, tcpu);
2106 
2107 		return &cpu_rq(cpu)->scx.local_dsq;
2108 	}
2109 
2110 	if (dsq_id == SCX_DSQ_GLOBAL)
2111 		dsq = find_global_dsq(sch, tcpu);
2112 	else
2113 		dsq = find_user_dsq(sch, dsq_id);
2114 
2115 	/*
2116 	 * Built-in DSQs are never inserted into dsq_hash, so REJECT and RESCUE
2117 	 * hit the error below. They cannot be reached with an ID.
2118 	 */
2119 	if (unlikely(!dsq)) {
2120 		scx_error(sch, "non-existent DSQ 0x%llx", dsq_id);
2121 		return find_global_dsq(sch, tcpu);
2122 	}
2123 
2124 	return dsq;
2125 }
2126 
2127 static void mark_direct_dispatch(struct scx_sched *sch,
2128 				 struct task_struct *ddsp_task,
2129 				 struct task_struct *p, u64 dsq_id,
2130 				 u64 slice, u64 vtime, u64 enq_flags)
2131 {
2132 	/*
2133 	 * Mark that dispatch already happened from ops.select_cpu() or
2134 	 * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value
2135 	 * which can never match a valid task pointer.
2136 	 */
2137 	__this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH));
2138 
2139 	/* @p must match the task on the enqueue path */
2140 	if (unlikely(p != ddsp_task)) {
2141 		if (IS_ERR(ddsp_task))
2142 			scx_error(sch, "%s[%d] already direct-dispatched",
2143 				  p->comm, p->pid);
2144 		else
2145 			scx_error(sch, "scheduling for %s[%d] but trying to direct-dispatch %s[%d]",
2146 				  ddsp_task->comm, ddsp_task->pid,
2147 				  p->comm, p->pid);
2148 		return;
2149 	}
2150 
2151 	WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID);
2152 	WARN_ON_ONCE(p->scx.ddsp_enq_flags);
2153 
2154 	p->scx.ddsp_slice = slice;
2155 	p->scx.ddsp_vtime = vtime;
2156 	p->scx.ddsp_dsq_id = dsq_id;
2157 	p->scx.ddsp_enq_flags = enq_flags;
2158 }
2159 
2160 /*
2161  * Clear @p direct dispatch state when leaving the scheduler.
2162  *
2163  * Direct dispatch state must be cleared in the following cases:
2164  *  - direct_dispatch(): cleared on the synchronous enqueue path, deferred
2165  *    dispatch keeps the state until consumed
2166  *  - process_ddsp_deferred_locals(): cleared after consuming deferred state,
2167  *  - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch
2168  *    verdict is ignored (local/global/bypass)
2169  *  - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering
2170  *    deferred cancellation and holding_cpu races
2171  *  - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by
2172  *    the scx_bypass() loop, so that stale state is not reused by a subsequent
2173  *    scheduler instance
2174  */
2175 static inline void clear_direct_dispatch(struct task_struct *p)
2176 {
2177 	p->scx.ddsp_dsq_id = SCX_DSQ_INVALID;
2178 	p->scx.ddsp_enq_flags = 0;
2179 }
2180 
2181 static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
2182 			    u64 enq_flags)
2183 {
2184 	struct rq *rq = task_rq(p);
2185 	struct scx_dispatch_q *dsq =
2186 		find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
2187 	u64 ddsp_enq_flags, slice, vtime;
2188 
2189 	p->scx.ddsp_enq_flags |= enq_flags;
2190 
2191 	/*
2192 	 * We are in the enqueue path with @rq locked and pinned, and thus can't
2193 	 * double lock a remote rq and enqueue to its local DSQ. For
2194 	 * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer
2195 	 * the enqueue so that it's executed when @rq can be unlocked.
2196 	 */
2197 	if (dsq->id == SCX_DSQ_LOCAL && dsq != &rq->scx.local_dsq) {
2198 		unsigned long opss;
2199 
2200 		opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK;
2201 
2202 		switch (opss & SCX_OPSS_STATE_MASK) {
2203 		case SCX_OPSS_NONE:
2204 			break;
2205 		case SCX_OPSS_QUEUEING:
2206 			/*
2207 			 * As @p was never passed to the BPF side, _release is
2208 			 * not strictly necessary. Still do it for consistency.
2209 			 */
2210 			atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
2211 			break;
2212 		default:
2213 			WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()",
2214 				  p->comm, p->pid, opss);
2215 			atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
2216 			break;
2217 		}
2218 
2219 		WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
2220 		list_add_tail(&p->scx.dsq_list.node,
2221 			      &rq->scx.ddsp_deferred_locals);
2222 		schedule_deferred_locked(rq);
2223 		return;
2224 	}
2225 
2226 	ddsp_enq_flags = p->scx.ddsp_enq_flags;
2227 	slice = p->scx.ddsp_slice;
2228 	vtime = p->scx.ddsp_vtime;
2229 	clear_direct_dispatch(p);
2230 
2231 	scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
2232 			     ddsp_enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
2233 }
2234 
2235 bool scx_rq_online(struct rq *rq)
2236 {
2237 	/*
2238 	 * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE is set
2239 	 * after the root's ops.cpu_online() and cleared when the cpu goes down.
2240 	 * cpu_active() test guarantees that, if this function returns %true,
2241 	 * %SCX_RQ_ONLINE will stay set until the current scheduling operation
2242 	 * is complete even if we aren't locking @rq.
2243 	 */
2244 	return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));
2245 }
2246 
2247 void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
2248 			 int sticky_cpu)
2249 {
2250 	enum {
2251 		ENQ_ACTION_NONE,
2252 		ENQ_ACTION_DIRECT,
2253 		ENQ_ACTION_LOCAL_NOREFILL,
2254 		ENQ_ACTION_ENQUEUE,
2255 	} action = ENQ_ACTION_NONE;
2256 	struct scx_sched *sch = scx_task_sched(p);
2257 	struct task_struct **ddsp_taskp;
2258 	struct scx_dispatch_q *dsq;
2259 	unsigned long qseq;
2260 
2261 	WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
2262 
2263 	/* internal movements - rq migration / RESTORE */
2264 	if (sticky_cpu == cpu_of(rq))
2265 		goto local_norefill;
2266 
2267 	/*
2268 	 * Clear persistent TASK_IMMED for fresh enqueues, see dsq_inc_nr().
2269 	 * Note that exiting and migration-disabled tasks that skip
2270 	 * ops.enqueue() below will lose IMMED protection unless
2271 	 * %SCX_OPS_ENQ_EXITING / %SCX_OPS_ENQ_MIGRATION_DISABLED are set.
2272 	 */
2273 	p->scx.flags &= ~SCX_TASK_IMMED;
2274 
2275 	/*
2276 	 * If !scx_rq_online(), we already told the BPF scheduler that the CPU
2277 	 * is offline and are just running the hotplug path. Don't bother the
2278 	 * BPF scheduler.
2279 	 */
2280 	if (!scx_rq_online(rq))
2281 		goto local;
2282 
2283 	if (scx_bypassing(sch, cpu_of(rq))) {
2284 		__scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1);
2285 		goto bypass;
2286 	}
2287 
2288 	/*
2289 	 * A task reenqueued too many times without running means the scheduler
2290 	 * keeps re-deciding a placement it can't honor, e.g. re-inserting to a
2291 	 * cid it lacks caps on. Eject the owning scheduler and strand the task
2292 	 * to be picked up during sched exit.
2293 	 *
2294 	 * Bypass guarantees that every runnable task keeps running, and the
2295 	 * disable relies on that to take sleeping locks. The kernel places the
2296 	 * tasks of a bypassing scheduler and those on an offline rq, and
2297 	 * counting their reenqueues could strand a task under bypass and
2298 	 * deadlock the disable on a lock the task holds, so both tests stay
2299 	 * above the count.
2300 	 */
2301 	if (enq_flags & SCX_ENQ_REENQ) {
2302 		if (++p->scx.reenq_cnt > 1)
2303 			__scx_add_event(sch, SCX_EV_REENQ_REPEAT, 1);
2304 
2305 		if (unlikely(p->scx.reenq_cnt > SCX_REENQ_MAX_REPEAT)) {
2306 			__scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq),
2307 				   "%s[%d] reenqueued %u times without running",
2308 				   p->comm, p->pid, p->scx.reenq_cnt);
2309 			goto out;
2310 		}
2311 	}
2312 
2313 	if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
2314 		goto direct;
2315 
2316 	/*
2317 	 * A full wakeup enqueue happens before is_blocked is cleared. Don't
2318 	 * report it as another blocked-donor admission.
2319 	 */
2320 	if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked &&
2321 	    !(enq_flags & SCX_ENQ_WAKEUP)) {
2322 		enq_flags |= SCX_ENQ_BLOCKED;
2323 	} else {
2324 		/* see %SCX_OPS_ENQ_EXITING */
2325 		if (unlikely(p->flags & PF_EXITING) &&
2326 		    !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) {
2327 			__scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1);
2328 			enq_flags |= SCX_ENQ_RESCUE;	/* avoid looping on cap rejection */
2329 			goto local;
2330 		}
2331 
2332 		/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
2333 		if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
2334 		    is_migration_disabled(p)) {
2335 			__scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
2336 			goto local;
2337 		}
2338 	}
2339 
2340 	if (unlikely(!SCX_HAS_OP(sch, enqueue)))
2341 		goto global;
2342 
2343 	/*
2344 	 * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the
2345 	 * per-task qseq counter where the QSEQ field wraps and skip 0, which is
2346 	 * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING.
2347 	 */
2348 	p->scx.ops_qseq = ((p->scx.ops_qseq + 1) &
2349 			   (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1;
2350 	qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT;
2351 
2352 	WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
2353 	atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq);
2354 
2355 	ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
2356 	WARN_ON_ONCE(*ddsp_taskp);
2357 	*ddsp_taskp = p;
2358 
2359 	SCX_CALL_OP_TASK(sch, enqueue, rq, p, enq_flags);
2360 
2361 	*ddsp_taskp = NULL;
2362 	if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
2363 		goto direct;
2364 
2365 	/*
2366 	 * Task is now in BPF scheduler's custody. Set %SCX_TASK_IN_CUSTODY
2367 	 * so ops.dequeue() is called when it leaves custody.
2368 	 */
2369 	p->scx.flags |= SCX_TASK_IN_CUSTODY;
2370 
2371 	/*
2372 	 * If not directly dispatched, QUEUEING isn't clear yet and dispatch or
2373 	 * dequeue may be waiting. The store_release matches their load_acquire.
2374 	 */
2375 	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq);
2376 	goto out;
2377 
2378 direct:
2379 	action = ENQ_ACTION_DIRECT;
2380 	goto out;
2381 local_norefill:
2382 	action = ENQ_ACTION_LOCAL_NOREFILL;
2383 	goto out;
2384 local:
2385 	dsq = &rq->scx.local_dsq;
2386 	goto enqueue;
2387 global:
2388 	dsq = find_global_dsq(sch, task_cpu(p));
2389 	goto enqueue;
2390 bypass:
2391 	dsq = bypass_enq_target_dsq(sch, rq, p, &enq_flags);
2392 	goto enqueue;
2393 
2394 enqueue:
2395 	action = ENQ_ACTION_ENQUEUE;
2396 out:
2397 	/* The reason is input to ops.enqueue(), not to the resulting placement. */
2398 	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
2399 
2400 	switch (action) {
2401 	case ENQ_ACTION_DIRECT:
2402 		direct_dispatch(sch, p, enq_flags);
2403 		break;
2404 	case ENQ_ACTION_LOCAL_NOREFILL:
2405 		scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
2406 				     enq_flags);
2407 		break;
2408 	case ENQ_ACTION_ENQUEUE:
2409 		refill_task_slice_dfl(sch, p);
2410 		clear_direct_dispatch(p);
2411 		scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
2412 		break;
2413 	case ENQ_ACTION_NONE:
2414 		break;
2415 	}
2416 }
2417 
2418 static bool task_runnable(const struct task_struct *p)
2419 {
2420 	return !list_empty(&p->scx.runnable_node);
2421 }
2422 
2423 static void set_task_runnable(struct rq *rq, struct task_struct *p)
2424 {
2425 	lockdep_assert_rq_held(rq);
2426 
2427 	if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) {
2428 		p->scx.runnable_at = jiffies;
2429 		p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT;
2430 	}
2431 
2432 	/*
2433 	 * list_add_tail() must be used. scx_bypass() and drain_offline_rq()
2434 	 * depend on tasks being appended to the runnable_list.
2435 	 */
2436 	list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
2437 
2438 	/*
2439 	 * Record the rq @p is runnable on, maintained under the rq lock so it
2440 	 * stays valid unlike task_cpu(), which a remote wakeup can move under
2441 	 * pi_lock alone.
2442 	 */
2443 	WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq));
2444 }
2445 
2446 static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at)
2447 {
2448 	list_del_init(&p->scx.runnable_node);
2449 	WRITE_ONCE(p->scx.runnable_cpu, -1);
2450 	if (reset_runnable_at) {
2451 		p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
2452 		p->scx.reenq_cnt = 0;
2453 	}
2454 }
2455 
2456 static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_flags)
2457 {
2458 	struct scx_sched *sch = scx_task_sched(p);
2459 	int sticky_cpu = p->scx.sticky_cpu;
2460 	u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags;
2461 
2462 	/*
2463 	 * An SCX-internal migration ends on arrival. Clear sticky_cpu so @p can
2464 	 * leave custody when inserted into the destination DSQ.
2465 	 */
2466 	if (sticky_cpu >= 0)
2467 		p->scx.sticky_cpu = -1;
2468 
2469 	/*
2470 	 * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue
2471 	 * returns. Only the core's wakeup path delivers one. The flags stashed
2472 	 * for a remote activation may carry the wakeup bit without it.
2473 	 */
2474 	if (core_enq_flags & ENQUEUE_WAKEUP)
2475 		rq->scx.flags |= SCX_RQ_IN_WAKEUP;
2476 
2477 	/*
2478 	 * Restoring the current scheduling context will be immediately followed
2479 	 * by set_next_task_scx() which expects the task to not be on the BPF
2480 	 * scheduler as tasks can only start running through local DSQs. Force
2481 	 * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark
2482 	 * IGNORE_CAPS to force entry into the local DSQ.
2483 	 */
2484 	if (unlikely(enq_flags & ENQUEUE_RESTORE) &&
2485 	    task_current_donor(rq, p)) {
2486 		sticky_cpu = cpu_of(rq);
2487 		enq_flags |= SCX_ENQ_IGNORE_CAPS;
2488 	}
2489 
2490 	if (p->scx.flags & SCX_TASK_QUEUED) {
2491 		WARN_ON_ONCE(!task_runnable(p));
2492 		goto out;
2493 	}
2494 
2495 	set_task_runnable(rq, p);
2496 	p->scx.flags |= SCX_TASK_QUEUED;
2497 	rq->scx.nr_running++;
2498 	add_nr_running(rq, 1);
2499 
2500 	if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p))
2501 		SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags);
2502 
2503 	/* Start dl_server if this is the first task being enqueued */
2504 	if (rq->scx.nr_running == 1)
2505 		dl_server_start(&rq->ext_server);
2506 
2507 	scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu);
2508 out:
2509 	rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
2510 
2511 	if ((enq_flags & SCX_ENQ_CPU_SELECTED) &&
2512 	    unlikely(cpu_of(rq) != p->scx.selected_cpu))
2513 		__scx_add_event(sch, SCX_EV_SELECT_CPU_FALLBACK, 1);
2514 }
2515 
2516 static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
2517 {
2518 	struct scx_sched *sch = scx_task_sched(p);
2519 	unsigned long opss;
2520 
2521 	/*
2522 	 * Don't reset runnable_at. If @p is waiting, the wait continues across
2523 	 * temporary dequeues, e.g. for a migration or an attribute change.
2524 	 * scx_reset_task_fields() ends it if @p leaves its sched's control.
2525 	 * scx_bypass() restarts it when its requeue can change the sched
2526 	 * running @p.
2527 	 */
2528 	clr_task_runnable(p, false);
2529 
2530 retry:
2531 	/* acquire ensures that we see the preceding updates on QUEUED */
2532 	opss = atomic_long_read_acquire(&p->scx.ops_state);
2533 
2534 	switch (opss & SCX_OPSS_STATE_MASK) {
2535 	case SCX_OPSS_NONE:
2536 		break;
2537 	case SCX_OPSS_QUEUEING:
2538 		/*
2539 		 * QUEUEING is started and finished while holding @p's rq lock.
2540 		 * As we're holding the rq lock now, we shouldn't see QUEUEING.
2541 		 */
2542 		BUG();
2543 	case SCX_OPSS_QUEUED:
2544 		/*
2545 		 * A queued task must always be in BPF scheduler's custody. If
2546 		 * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another
2547 		 * CPU has already passed task_leave_custody() (which clears the
2548 		 * flag), but has not yet written SCX_OPSS_NONE. That final
2549 		 * store does not require this rq's lock, so retrying with
2550 		 * cpu_relax() is bounded: we will observe NONE (or DISPATCHING,
2551 		 * handled by the fallthrough) on a subsequent iteration.
2552 		 */
2553 		if (unlikely(!(READ_ONCE(p->scx.flags) & SCX_TASK_IN_CUSTODY))) {
2554 			cpu_relax();
2555 			goto retry;
2556 		}
2557 
2558 		if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
2559 					    SCX_OPSS_NONE))
2560 			break;
2561 		fallthrough;
2562 	case SCX_OPSS_DISPATCHING:
2563 		/*
2564 		 * If @p is being dispatched from the BPF scheduler to a DSQ,
2565 		 * wait for the transfer to complete so that @p doesn't get
2566 		 * added to its DSQ after dequeueing is complete.
2567 		 *
2568 		 * As we're waiting on DISPATCHING with the rq locked, the
2569 		 * dispatching side shouldn't try to lock the rq while
2570 		 * DISPATCHING is set. See dispatch_to_local_dsq().
2571 		 *
2572 		 * DISPATCHING shouldn't have qseq set and control can reach
2573 		 * here with NONE @opss from the above QUEUED case block.
2574 		 * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss.
2575 		 */
2576 		wait_ops_state(p, SCX_OPSS_DISPATCHING);
2577 		BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
2578 		break;
2579 	}
2580 
2581 	/*
2582 	 * Call ops.dequeue() if the task is still in BPF custody.
2583 	 *
2584 	 * The code that clears ops_state to %SCX_OPSS_NONE does not always
2585 	 * clear %SCX_TASK_IN_CUSTODY: in dispatch_to_local_dsq(), when
2586 	 * we're moving a task that was in %SCX_OPSS_DISPATCHING to a
2587 	 * remote CPU's local DSQ, we only set ops_state to %SCX_OPSS_NONE
2588 	 * so that a concurrent dequeue can proceed, but we clear
2589 	 * %SCX_TASK_IN_CUSTODY only when we later enqueue or move the
2590 	 * task. So we can see NONE + IN_CUSTODY here and we must handle
2591 	 * it. Similarly, after waiting on %SCX_OPSS_DISPATCHING we see
2592 	 * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until
2593 	 * it is enqueued on the destination.
2594 	 */
2595 	if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
2596 		SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
2597 }
2598 
2599 static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags)
2600 {
2601 	struct scx_sched *sch = scx_task_sched(p);
2602 	u64 deq_flags = core_deq_flags;
2603 
2604 	/*
2605 	 * Set %SCX_DEQ_SCHED_CHANGE when the dequeue is due to a property
2606 	 * change (not sleep).
2607 	 */
2608 	if (!(deq_flags & DEQUEUE_SLEEP))
2609 		deq_flags |= SCX_DEQ_SCHED_CHANGE;
2610 
2611 	if (!(p->scx.flags & SCX_TASK_QUEUED)) {
2612 		WARN_ON_ONCE(task_runnable(p));
2613 		return true;
2614 	}
2615 
2616 	ops_dequeue(rq, p, deq_flags);
2617 
2618 	/*
2619 	 * A current scheduling context which is going off @rq first gets
2620 	 * dequeued and then stops running. As we want running <-> stopping
2621 	 * transitions to be contained within runnable <-> quiescent transitions,
2622 	 * trigger ->stopping() early here instead of in put_prev_task_scx().
2623 	 *
2624 	 * @p may go through multiple stopping <-> running transitions between
2625 	 * here and put_prev_task_scx() if task attribute changes occur while
2626 	 * dispatch_one() leaves @rq unlocked. However, they don't contain any
2627 	 * information meaningful to the BPF scheduler and can be suppressed by
2628 	 * skipping the callbacks if the task is !QUEUED.
2629 	 */
2630 	if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) {
2631 		if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) {
2632 			update_curr_scx(rq);
2633 			if (SCX_HAS_OP(sch, stopping))
2634 				SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
2635 		}
2636 		p->scx.flags &= ~SCX_TASK_RUN_TRACKED;
2637 	}
2638 
2639 	if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p))
2640 		SCX_CALL_OP_TASK(sch, quiescent, rq, p, deq_flags);
2641 
2642 	if (deq_flags & SCX_DEQ_SLEEP)
2643 		p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP;
2644 	else
2645 		p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP;
2646 
2647 	p->scx.flags &= ~SCX_TASK_QUEUED;
2648 	rq->scx.nr_running--;
2649 	sub_nr_running(rq, 1);
2650 
2651 	scx_dispatch_dequeue(rq, p);
2652 	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
2653 
2654 	/* see scx_task_slice_ended() for the save/restore exception */
2655 	if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p)))
2656 		scx_task_slice_ended(rq, p);
2657 
2658 	clear_direct_dispatch(p);
2659 	return true;
2660 }
2661 
2662 static void yield_task_scx(struct rq *rq)
2663 {
2664 	struct task_struct *p = rq->donor;
2665 	struct scx_sched *sch = scx_task_sched(p);
2666 
2667 	/* a yield gives the slice up */
2668 	scx_task_slice_ended(rq, p);
2669 
2670 	if (SCX_HAS_OP(sch, yield))
2671 		SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL);
2672 	else
2673 		scx_set_task_slice(p, 0);
2674 }
2675 
2676 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
2677 {
2678 	struct task_struct *from = rq->donor;
2679 	struct scx_sched *sch = scx_task_sched(from);
2680 
2681 	/* like a plain yield, giving the slice up ends the protection */
2682 	scx_task_slice_ended(rq, from);
2683 
2684 	if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to))
2685 		return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to);
2686 	else
2687 		return false;
2688 }
2689 
2690 static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_flags)
2691 {
2692 	/*
2693 	 * Preemption between SCX tasks is implemented by resetting the victim
2694 	 * task's slice to 0 and triggering reschedule on the target CPU.
2695 	 *
2696 	 * A retained proxy donor can wake through ttwu_runnable() without another
2697 	 * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so
2698 	 * that ops.dispatch() can reconsider the task after ttwu_runnable() clears
2699 	 * is_blocked. A full wakeup activation has already enqueued the task and
2700 	 * doesn't need the additional reschedule.
2701 	 */
2702 	if (p->sched_class == &ext_sched_class) {
2703 		if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) &&
2704 		    p->is_blocked) {
2705 			struct scx_sched *sch = scx_task_sched(p);
2706 
2707 			if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED))
2708 				resched_curr(rq);
2709 		}
2710 		return;
2711 	}
2712 
2713 	/*
2714 	 * Getting preempted by a higher-priority class. Reenqueue IMMED tasks.
2715 	 * This captures all preemption cases including:
2716 	 *
2717 	 * - A SCX task is currently running.
2718 	 *
2719 	 * - @rq is waking from idle due to a SCX task waking to it.
2720 	 *
2721 	 * - A higher-priority wakes up while SCX dispatch is in progress.
2722 	 */
2723 	if (rq->scx.nr_immed)
2724 		scx_schedule_reenq_local(rq, 0);
2725 }
2726 
2727 void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,
2728 				      u64 enq_flags, struct rq *dst_rq)
2729 {
2730 	struct scx_dispatch_q *dst_dsq;
2731 
2732 	lockdep_assert_rq_held(dst_rq);
2733 
2734 	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
2735 	scx_wait_dispatching(p);
2736 
2737 	dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
2738 
2739 	if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY))
2740 		dsq_insert_head(dst_dsq, p);
2741 	else
2742 		list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
2743 
2744 	dsq_inc_nr(dst_dsq, p, enq_flags);
2745 	p->scx.dsq = dst_dsq;
2746 
2747 	rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags);
2748 }
2749 
2750 /**
2751  * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ
2752  * @sch: scheduler placing @p
2753  * @p: task to move
2754  * @enq_flags: %SCX_ENQ_*
2755  * @src_rq: rq to move the task from, locked on entry, released on return
2756  * @dst_rq: rq to move the task into, locked on return
2757  *
2758  * Move @p which is currently on @src_rq to @dst_rq's local DSQ.
2759  */
2760 static void move_remote_task_to_local_dsq(struct scx_sched *sch,
2761 					  struct task_struct *p, u64 enq_flags,
2762 					  struct rq *src_rq, struct rq *dst_rq)
2763 {
2764 	lockdep_assert_rq_held(src_rq);
2765 
2766 	/*
2767 	 * Set sticky_cpu before deactivate_task() to properly mark the
2768 	 * beginning of an SCX-internal migration.
2769 	 */
2770 	p->scx.sticky_cpu = cpu_of(dst_rq);
2771 	deactivate_task(src_rq, p, 0);
2772 	set_task_cpu(p, cpu_of(dst_rq));
2773 
2774 	switch_rq_lock(src_rq, dst_rq);
2775 
2776 	/*
2777 	 * activate_task() below truncates enq_flags to 32 bits and re-derives
2778 	 * @p's owner, dropping our scx flags and the placing @sch. We own @rq,
2779 	 * so stash both across the call. The enqueue reads them back, keeping
2780 	 * the scx flags and checking caps against the placer, not the owner.
2781 	 */
2782 	WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr));
2783 	WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags ||
2784 		     dst_rq->scx.remote_activate_sch);
2785 	dst_rq->scx.remote_activate_enq_flags = enq_flags;
2786 	dst_rq->scx.remote_activate_sch = sch;
2787 	activate_task(dst_rq, p, 0);
2788 	dst_rq->scx.remote_activate_enq_flags = 0;
2789 	dst_rq->scx.remote_activate_sch = NULL;
2790 }
2791 
2792 /*
2793  * Similar to kernel/sched/core.c::is_cpu_allowed(). However, there are two
2794  * differences:
2795  *
2796  * - is_cpu_allowed() asks "Can this task run on this CPU?" while
2797  *   task_can_run_on_remote_rq() asks "Can the BPF scheduler migrate the task to
2798  *   this CPU?".
2799  *
2800  *   While migration is disabled, is_cpu_allowed() has to say "yes" as the task
2801  *   must be allowed to finish on the CPU that it's currently on regardless of
2802  *   the CPU state. However, task_can_run_on_remote_rq() must say "no" as the
2803  *   BPF scheduler shouldn't attempt to migrate a task which has migration
2804  *   disabled.
2805  *
2806  * - The BPF scheduler is bypassed while the rq is offline and we can always say
2807  *   no to the BPF scheduler initiated migrations while offline.
2808  *
2809  * The caller must ensure that @p and @rq are on different CPUs. If @enforce is
2810  * true, report violations attributable to BPF-directed migrations. The caller
2811  * must hold @p's rq lock to avoid reporting a transient race as a scheduler
2812  * error.
2813  */
2814 static bool task_can_run_on_remote_rq(struct scx_sched *sch,
2815 				      struct task_struct *p, struct rq *rq,
2816 				      bool enforce)
2817 {
2818 	s32 cpu = cpu_of(rq);
2819 
2820 	if (enforce)
2821 		lockdep_assert_rq_held(task_rq(p));
2822 
2823 	WARN_ON_ONCE(task_cpu(p) == cpu);
2824 
2825 	/*
2826 	 * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling
2827 	 * context to its lock owner's rq. For example, if BPF places donor D on
2828 	 * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1
2829 	 * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared
2830 	 * DSQ, CPU0 could otherwise consume it and move it back, only for proxy
2831 	 * execution to return it to CPU1 again.
2832 	 *
2833 	 * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and
2834 	 * BPF may still choose its placement. An actively donating task is
2835 	 * rejected separately by task_proxy_running_or_donating().
2836 	 */
2837 	if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu)
2838 		return false;
2839 
2840 	/*
2841 	 * If @p has migration disabled, @p->cpus_ptr is updated to contain only
2842 	 * the pinned CPU in migrate_disable_switch() while @p is being switched
2843 	 * out. However, put_prev_task_scx() is called before @p->cpus_ptr is
2844 	 * updated and thus another CPU may see @p on a DSQ inbetween leading to
2845 	 * @p passing the below task_allowed_on_cpu() check while migration is
2846 	 * disabled.
2847 	 *
2848 	 * Test the migration disabled state first as the race window is narrow
2849 	 * and the BPF scheduler failing to check migration disabled state can
2850 	 * easily be masked if task_allowed_on_cpu() is done first.
2851 	 */
2852 	if (unlikely(is_migration_disabled(p))) {
2853 		if (enforce)
2854 			scx_error(sch, "SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d",
2855 				  p->comm, p->pid, task_cpu(p), cpu);
2856 		return false;
2857 	}
2858 
2859 	/*
2860 	 * We don't require the BPF scheduler to avoid dispatching to offline
2861 	 * CPUs mostly for convenience but also because CPUs can go offline
2862 	 * between scx_bpf_dsq_insert() calls and here. Trigger error iff the
2863 	 * picked CPU is outside the allowed mask.
2864 	 */
2865 	if (!task_allowed_on_cpu(p, cpu)) {
2866 		if (enforce)
2867 			scx_error(sch, "SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]",
2868 				  cpu, p->comm, p->pid);
2869 		return false;
2870 	}
2871 
2872 	if (!scx_rq_online(rq)) {
2873 		if (enforce)
2874 			__scx_add_event(sch, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE, 1);
2875 		return false;
2876 	}
2877 
2878 	return true;
2879 }
2880 
2881 /*
2882  * Proxy execution can change @p's execution and migration-disabled state
2883  * without touching its DSQ entry or clearing holding_cpu. Check those states
2884  * with @p's rq locked. Without proxy execution, the holding_cpu handshake is
2885  * sufficient and this must not affect the existing migration path.
2886  *
2887  * A BPF-directed transfer to a remote local DSQ performs a normal task
2888  * migration and thus cannot move a migration-disabled task. In contrast,
2889  * proxy_migrate_task() moves only a blocked donor's scheduling context towards
2890  * the mutex owner and preserves its execution home in wake_cpu. The latter is
2891  * therefore allowed even when the donor is migration-disabled.
2892  */
2893 static bool task_proxy_running_or_donating(struct task_struct *p)
2894 {
2895 	struct rq *src_rq = task_rq(p);
2896 
2897 	lockdep_assert_rq_held(src_rq);
2898 
2899 	if (!sched_proxy_exec())
2900 		return false;
2901 
2902 	/* @p may be rq->curr under another task's scheduling context. */
2903 	if (task_on_cpu(src_rq, p))
2904 		return true;
2905 
2906 	/* Don't move an active scheduling context off its source rq. */
2907 	if (task_current_donor(src_rq, p))
2908 		return true;
2909 
2910 	return false;
2911 }
2912 
2913 static bool task_proxy_unsafe_to_move(struct task_struct *p)
2914 {
2915 	if (!sched_proxy_exec())
2916 		return false;
2917 
2918 	return task_proxy_running_or_donating(p) || is_migration_disabled(p);
2919 }
2920 
2921 /*
2922  * Park a task whose remote transfer raced with proxy execution. Reenqueueing
2923  * from the source rq makes the task's owning scheduler choose its placement
2924  * again and preserves sub-scheduler containment.
2925  */
2926 static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq,
2927 				  struct task_struct *p, u64 enq_flags)
2928 {
2929 	lockdep_assert_rq_held(rq);
2930 	WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
2931 
2932 	p->scx.flags |= SCX_TASK_REENQ_PROXY;
2933 	scx_divert_strip_flags(p, &enq_flags);
2934 
2935 	scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags);
2936 }
2937 
2938 /**
2939  * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock
2940  * @p: target task
2941  * @dsq: locked DSQ @p is currently on
2942  * @locked_rq: currently locked rq
2943  * @src_rq: rq @p is currently on, stable with @dsq locked
2944  *
2945  * Called with @dsq and @locked_rq locked. We want to move @p to a different DSQ,
2946  * including any local DSQ, but are not locking @src_rq. Locking @src_rq is
2947  * required when transferring into a local DSQ. Even when transferring into a
2948  * non-local DSQ, it's better to use the same mechanism to protect against
2949  * dequeues and maintain the invariant that @p->scx.dsq can only change while
2950  * @src_rq is locked, which e.g. scx_dump_task() depends on.
2951  *
2952  * We want to grab @src_rq but that can deadlock if we try while locking @dsq,
2953  * so we want to unlink @p from @dsq, drop its lock and then lock @src_rq. As
2954  * this may race with dequeue, which can't drop the rq lock or fail, do a little
2955  * dancing from our side.
2956  *
2957  * @p->scx.holding_cpu is set to this CPU before @dsq is unlocked. If @p gets
2958  * dequeued after we unlock @dsq but before locking @src_rq, the holding_cpu
2959  * would be cleared to -1. While other cpus may have updated it to different
2960  * values afterwards, as this operation can't be preempted or recurse, the
2961  * holding_cpu can never become this CPU again before we're done. Thus, we can
2962  * tell whether we lost to dequeue by testing whether the holding_cpu still
2963  * points to this CPU. See scx_dispatch_dequeue() for the counterpart.
2964  *
2965  * On return, @dsq is unlocked and @src_rq is locked. Return %true if @p is
2966  * still valid and detached from @dsq, with holding_cpu cleared. Return %false
2967  * if @p was lost to a dequeue.
2968  */
2969 static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
2970 					  struct scx_dispatch_q *dsq,
2971 					  struct rq *locked_rq,
2972 					  struct rq *src_rq)
2973 {
2974 	s32 cpu = raw_smp_processor_id();
2975 
2976 	lockdep_assert_held(&dsq->lock);
2977 	lockdep_assert_rq_held(locked_rq);
2978 
2979 	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
2980 	scx_task_unlink_from_dsq(p, dsq);
2981 	p->scx.holding_cpu = cpu;
2982 
2983 	raw_spin_unlock(&dsq->lock);
2984 	switch_rq_lock(locked_rq, src_rq);
2985 
2986 	/* task_rq couldn't have changed if we're still the holding cpu */
2987 	if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) {
2988 		/*
2989 		 * Keep ->dsq set until we own @src_rq so that a racing dequeue
2990 		 * can find @dsq and clear holding_cpu. Once we own @src_rq, the
2991 		 * handshake is over. Clear ->dsq so that deactivate_task()
2992 		 * takes the !dsq path instead of retaking @dsq->lock. Clear
2993 		 * holding_cpu with it so that no caller has to.
2994 		 */
2995 		p->scx.dsq = NULL;
2996 		p->scx.holding_cpu = -1;
2997 		scx_wait_dispatching(p);
2998 		return true;
2999 	}
3000 
3001 	return false;
3002 }
3003 
3004 static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
3005 				struct task_struct *p, u64 enq_flags,
3006 				struct scx_dispatch_q *dsq, struct rq *src_rq)
3007 {
3008 	if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) {
3009 		/*
3010 		 * Proxy execution may have changed @p's running or
3011 		 * migration-disabled state while switching rq locks without
3012 		 * clearing holding_cpu. Park it on the source rq and let its
3013 		 * owning scheduler choose its placement again.
3014 		 */
3015 		if (unlikely(task_proxy_unsafe_to_move(p))) {
3016 			p->scx.dsq = NULL;
3017 			scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
3018 			switch_rq_lock(src_rq, this_rq);
3019 			return false;
3020 		}
3021 
3022 		move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq);
3023 		return true;
3024 	} else {
3025 		switch_rq_lock(src_rq, this_rq);
3026 		return false;
3027 	}
3028 }
3029 
3030 /**
3031  * move_task_between_dsqs() - Move a task from one DSQ to another
3032  * @sch: scx_sched being operated on
3033  * @p: target task
3034  * @enq_flags: %SCX_ENQ_*
3035  * @src_dsq: DSQ @p is currently on, must not be a local DSQ
3036  * @dst_dsq: DSQ @p is being moved to, can be any DSQ
3037  *
3038  * Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local
3039  * DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq
3040  * will change. As @p's task_rq is locked, this function doesn't need to use the
3041  * holding_cpu mechanism.
3042  *
3043  * On return, @src_dsq is unlocked and only @p's new task_rq, which is the
3044  * return value, is locked.
3045  */
3046 static struct rq *move_task_between_dsqs(struct scx_sched *sch,
3047 					 struct task_struct *p, u64 enq_flags,
3048 					 struct scx_dispatch_q *src_dsq,
3049 					 struct scx_dispatch_q *dst_dsq)
3050 {
3051 	struct rq *src_rq = task_rq(p), *dst_rq;
3052 
3053 	BUG_ON(src_dsq->id == SCX_DSQ_LOCAL);
3054 	lockdep_assert_held(&src_dsq->lock);
3055 	lockdep_assert_rq_held(src_rq);
3056 
3057 	scx_wait_dispatching(p);
3058 
3059 	if (dst_dsq->id == SCX_DSQ_LOCAL) {
3060 		dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
3061 		/*
3062 		 * Unlike the rq-lock handoff paths, @src_rq has been locked
3063 		 * throughout this operation. Only active proxy state can race the
3064 		 * move here; let the enforcing check below diagnose an ordinary
3065 		 * migration-disabled task.
3066 		 */
3067 		if (src_rq != dst_rq &&
3068 		    unlikely(task_proxy_running_or_donating(p))) {
3069 			dispatch_dequeue_locked(p, src_dsq);
3070 			raw_spin_unlock(&src_dsq->lock);
3071 			scx_proxy_reject_task(sch, src_rq, p, enq_flags);
3072 			return src_rq;
3073 		}
3074 		if (src_rq != dst_rq &&
3075 		    unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
3076 			dst_dsq = find_global_dsq(sch, task_cpu(p));
3077 			dst_rq = src_rq;
3078 			enq_flags |= SCX_ENQ_GDSQ_FALLBACK;
3079 		}
3080 	} else {
3081 		/* no need to migrate if destination is a non-local DSQ */
3082 		dst_rq = src_rq;
3083 	}
3084 
3085 	/*
3086 	 * Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different
3087 	 * CPU, @p will be migrated.
3088 	 */
3089 	if (dst_dsq->id == SCX_DSQ_LOCAL) {
3090 		/* @p is going from a non-local DSQ to a local DSQ */
3091 		if (src_rq == dst_rq) {
3092 			scx_task_unlink_from_dsq(p, src_dsq);
3093 			raw_spin_unlock(&src_dsq->lock);
3094 			scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq);
3095 		} else {
3096 			dispatch_dequeue_locked(p, src_dsq);
3097 			raw_spin_unlock(&src_dsq->lock);
3098 			move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
3099 		}
3100 	} else {
3101 		/*
3102 		 * @p is going from a non-local DSQ to a non-local DSQ. As
3103 		 * $src_dsq is already locked, do an abbreviated dequeue.
3104 		 */
3105 		dispatch_dequeue_locked(p, src_dsq);
3106 		raw_spin_unlock(&src_dsq->lock);
3107 
3108 		scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags);
3109 	}
3110 
3111 	return dst_rq;
3112 }
3113 
3114 bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
3115 			    struct scx_dispatch_q *dsq, u64 enq_flags)
3116 {
3117 	struct task_struct *p;
3118 retry:
3119 	/*
3120 	 * The caller can't expect to successfully consume a task if the task's
3121 	 * addition to @dsq isn't guaranteed to be visible somehow. Test
3122 	 * @dsq->list without locking and skip if it seems empty.
3123 	 */
3124 	if (list_empty(&dsq->list))
3125 		return false;
3126 
3127 	raw_spin_lock(&dsq->lock);
3128 
3129 	nldsq_for_each_task(p, dsq) {
3130 		struct rq *task_rq = task_rq(p);
3131 
3132 		/*
3133 		 * This loop can lead to multiple lockup scenarios, e.g. the BPF
3134 		 * scheduler can put an enormous number of affinitized tasks into
3135 		 * a contended DSQ, or the outer retry loop can repeatedly race
3136 		 * against scx_bypass() dequeueing tasks from @dsq trying to put
3137 		 * the system into the bypass mode. This can easily live-lock the
3138 		 * machine. If aborting, exit from all non-bypass DSQs.
3139 		 */
3140 		if (unlikely(READ_ONCE(sch->aborting)) && dsq->id != SCX_DSQ_BYPASS)
3141 			break;
3142 
3143 		if (rq == task_rq) {
3144 			scx_task_unlink_from_dsq(p, dsq);
3145 			raw_spin_unlock(&dsq->lock);
3146 			scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq);
3147 			return true;
3148 		}
3149 
3150 		if (task_can_run_on_remote_rq(sch, p, rq, false)) {
3151 			if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq)))
3152 				return true;
3153 			goto retry;
3154 		}
3155 	}
3156 
3157 	raw_spin_unlock(&dsq->lock);
3158 	return false;
3159 }
3160 
3161 bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq)
3162 {
3163 	int node = cpu_to_node(cpu_of(rq));
3164 
3165 	return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0);
3166 }
3167 
3168 /**
3169  * dispatch_to_local_dsq - Dispatch a task to a local dsq
3170  * @sch: scx_sched being operated on
3171  * @rq: current rq which is locked
3172  * @dst_dsq: destination DSQ
3173  * @p: task to dispatch
3174  * @slice: slice carried by the insert verdict, 0 keeps the current value
3175  * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts
3176  * @enq_flags: %SCX_ENQ_*
3177  *
3178  * We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local
3179  * DSQ. This function performs all the synchronization dancing needed because
3180  * local DSQs are protected with rq locks.
3181  *
3182  * The caller must have exclusive ownership of @p (e.g. through
3183  * %SCX_OPSS_DISPATCHING).
3184  */
3185 static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
3186 				  struct scx_dispatch_q *dst_dsq, struct task_struct *p,
3187 				  u64 slice, u64 vtime, u64 enq_flags)
3188 {
3189 	struct rq *src_rq = task_rq(p);
3190 	struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
3191 	struct rq *locked_rq = rq;
3192 
3193 	/*
3194 	 * We're synchronized against dequeue through DISPATCHING. As @p can't
3195 	 * be dequeued, its task_rq and cpus_allowed are stable too.
3196 	 *
3197 	 * If dispatching to @rq that @p is already on, no lock dancing needed.
3198 	 */
3199 	if (rq == src_rq && rq == dst_rq) {
3200 		scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime,
3201 				     enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
3202 		return;
3203 	}
3204 
3205 	/*
3206 	 * @p is on a possibly remote @src_rq which we need to lock to move the
3207 	 * task. If dequeue is in progress, it'd be locking @src_rq and waiting
3208 	 * on DISPATCHING, so we can't grab @src_rq lock while holding
3209 	 * DISPATCHING.
3210 	 *
3211 	 * As DISPATCHING guarantees that @p is wholly ours, we can pretend that
3212 	 * we're moving from a DSQ and use the same mechanism - mark the task
3213 	 * under transfer with holding_cpu, release DISPATCHING and then follow
3214 	 * the same protocol. See unlink_dsq_and_switch_rq_lock().
3215 	 */
3216 	p->scx.holding_cpu = raw_smp_processor_id();
3217 
3218 	/* store_release ensures that dequeue sees the above */
3219 	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
3220 
3221 	/* switch to @src_rq lock */
3222 	if (locked_rq != src_rq) {
3223 		switch_rq_lock(locked_rq, src_rq);
3224 		locked_rq = src_rq;
3225 	}
3226 
3227 	/* task_rq couldn't have changed if we're still the holding cpu */
3228 	if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
3229 	    !WARN_ON_ONCE(src_rq != task_rq(p))) {
3230 		bool fallback = false;
3231 
3232 		/* the handshake is over, see unlink_dsq_and_switch_rq_lock() */
3233 		p->scx.holding_cpu = -1;
3234 
3235 		/*
3236 		 * If @p is staying on the same rq, there's no need to go
3237 		 * through the full deactivate/activate cycle. Optimize by
3238 		 * abbreviating move_remote_task_to_local_dsq().
3239 		 */
3240 		if (src_rq == dst_rq) {
3241 			scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
3242 					     slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE);
3243 		} else if (unlikely(task_proxy_unsafe_to_move(p))) {
3244 			fallback = true;
3245 			scx_proxy_reject_task(sch, src_rq, p, enq_flags);
3246 		} else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
3247 			fallback = true;
3248 			scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
3249 					     p, slice, vtime,
3250 					     enq_flags | SCX_ENQ_APPLY_SLICE |
3251 					     SCX_ENQ_GDSQ_FALLBACK);
3252 		} else {
3253 			apply_slice_vtime(p, slice, vtime, enq_flags);
3254 			move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
3255 			/* task has been moved to dst_rq, which is now locked */
3256 			locked_rq = dst_rq;
3257 		}
3258 
3259 		/* if the destination CPU is idle, wake it up */
3260 		if (!fallback && sched_class_above(p->sched_class,
3261 						      dst_rq->donor->sched_class))
3262 			resched_curr(dst_rq);
3263 	}
3264 
3265 	/* switch back to @rq lock */
3266 	if (locked_rq != rq)
3267 		switch_rq_lock(locked_rq, rq);
3268 }
3269 
3270 /**
3271  * finish_dispatch - Asynchronously finish dispatching a task
3272  * @sch: the scheduler
3273  * @rq: current rq which is locked
3274  * @p: task to finish dispatching
3275  * @qseq_at_dispatch: qseq when @p started getting dispatched
3276  * @dsq_id: destination DSQ ID
3277  * @slice: slice carried by the insert verdict, 0 keeps the current value
3278  * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts
3279  * @enq_flags: %SCX_ENQ_*
3280  *
3281  * Dispatching to local DSQs may need to wait for queueing to complete or
3282  * require rq lock dancing. As we don't wanna do either while inside
3283  * ops.dispatch() to avoid locking order inversion, we split dispatching into
3284  * two parts. scx_bpf_dsq_insert() which is called by ops.dispatch() records the
3285  * task and its qseq. Once ops.dispatch() returns, this function is called to
3286  * finish up.
3287  *
3288  * There is no guarantee that @p is still valid for dispatching or even that it
3289  * was valid in the first place. Make sure that the task is still owned by the
3290  * BPF scheduler and claim the ownership before dispatching.
3291  */
3292 static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p,
3293 			    unsigned long qseq_at_dispatch, u64 dsq_id,
3294 			    u64 slice, u64 vtime, u64 enq_flags)
3295 {
3296 	struct scx_dispatch_q *dsq;
3297 	unsigned long opss;
3298 
3299 retry:
3300 	/*
3301 	 * No need for _acquire here. @p is accessed only after a successful
3302 	 * try_cmpxchg to DISPATCHING.
3303 	 */
3304 	opss = atomic_long_read(&p->scx.ops_state);
3305 
3306 	switch (opss & SCX_OPSS_STATE_MASK) {
3307 	case SCX_OPSS_DISPATCHING:
3308 	case SCX_OPSS_NONE:
3309 		/* someone else already got to it */
3310 		return;
3311 	case SCX_OPSS_QUEUED:
3312 		/*
3313 		 * If qseq doesn't match, @p has gone through at least one
3314 		 * dispatch/dequeue and re-enqueue cycle between
3315 		 * scx_bpf_dsq_insert() and here and we have no claim on it.
3316 		 */
3317 		if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch)
3318 			return;
3319 
3320 		/* see SCX_EV_INSERT_NOT_OWNED definition */
3321 		if (unlikely(!scx_task_on_sched(sch, p))) {
3322 			__scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1);
3323 			return;
3324 		}
3325 
3326 		/*
3327 		 * While we know @p is accessible, we don't yet have a claim on
3328 		 * it - the BPF scheduler is allowed to dispatch tasks
3329 		 * spuriously and there can be a racing dequeue attempt. Let's
3330 		 * claim @p by atomically transitioning it from QUEUED to
3331 		 * DISPATCHING.
3332 		 */
3333 		if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
3334 						   SCX_OPSS_DISPATCHING)))
3335 			break;
3336 		goto retry;
3337 	case SCX_OPSS_QUEUEING:
3338 		/*
3339 		 * scx_do_enqueue_task() is in the process of transferring the
3340 		 * task to the BPF scheduler while holding @p's rq lock. As we
3341 		 * aren't holding any kernel or BPF resource that the enqueue
3342 		 * path may depend upon, it's safe to wait.
3343 		 */
3344 		wait_ops_state(p, opss);
3345 		goto retry;
3346 	}
3347 
3348 	BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED));
3349 
3350 	dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
3351 
3352 	if (dsq->id == SCX_DSQ_LOCAL)
3353 		dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
3354 	else
3355 		scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
3356 				     enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
3357 }
3358 
3359 void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
3360 {
3361 	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
3362 	u32 u;
3363 
3364 	for (u = 0; u < dspc->cursor; u++) {
3365 		struct scx_dsp_buf_ent *ent = &dspc->buf[u];
3366 
3367 		finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id,
3368 				ent->slice, ent->vtime, ent->enq_flags);
3369 	}
3370 
3371 	dspc->nr_tasks += dspc->cursor;
3372 	dspc->cursor = 0;
3373 }
3374 
3375 static inline void maybe_queue_balance_callback(struct rq *rq)
3376 {
3377 	lockdep_assert_rq_held(rq);
3378 
3379 	if (!(rq->scx.flags & SCX_RQ_BAL_CB_PENDING))
3380 		return;
3381 
3382 	queue_balance_callback(rq, &rq->scx.deferred_bal_cb,
3383 				deferred_bal_cb_workfn);
3384 
3385 	rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
3386 }
3387 
3388 static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev)
3389 {
3390 	struct scx_sched *root_sch = scx_root_protected_live();
3391 	enum scx_dsp_verdict verdict;
3392 	s32 cpu = cpu_of(rq);
3393 
3394 	lockdep_assert_rq_held(rq);
3395 	rq->scx.flags |= SCX_RQ_IN_DISPATCH;
3396 
3397 	scx_process_sync_ecaps(rq, prev);
3398 
3399 	if ((root_sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
3400 	    unlikely(rq->scx.cpu_released)) {
3401 		/*
3402 		 * If the previous sched_class for the current CPU was not SCX,
3403 		 * notify the BPF scheduler that it again has control of the
3404 		 * core. This callback complements ->cpu_release(), which is
3405 		 * emitted in switch_class().
3406 		 */
3407 		if (root_sch->ops.cpu_acquire)
3408 			SCX_CALL_OP(root_sch, cpu_acquire, rq, cpu, NULL);
3409 		rq->scx.cpu_released = false;
3410 	}
3411 
3412 	/* the ecaps sync may have dropped the rq lock */
3413 	if (scx_dsp_prev_stale(rq, prev)) {
3414 		verdict = SCX_DSP_RETRY;
3415 		goto has_tasks;
3416 	}
3417 
3418 	if (prev->sched_class == &ext_sched_class) {
3419 		update_curr_scx(rq);
3420 
3421 		/*
3422 		 * Under bypass, @prev's slice can't be trusted as a cpu-form
3423 		 * scheduler can write it directly from BPF. Skipping the keep
3424 		 * below is not enough as the post-dispatch test in
3425 		 * scx_dispatch_sched() would keep @prev on the same slice.
3426 		 *
3427 		 * Expire it as the bypass tick would, so that neither test
3428 		 * keeps @prev on it and @prev rotates through the bypass
3429 		 * enqueue. A kernel slice is left alone and the keep below
3430 		 * keeps @prev on it, see %SCX_TASK_KERNEL_SLICE.
3431 		 *
3432 		 * TODO: Once the cpu form is deprecated and the kfuncs refuse
3433 		 * slice raises from a bypassing scheduler, slices can be
3434 		 * trusted under bypass. This expiry, the bypass tick's zeroing
3435 		 * and the related slice handling can then be cleaned up.
3436 		 */
3437 		if (scx_bypassing(scx_task_sched(prev), cpu))
3438 			expire_task_slice(prev);
3439 
3440 		/*
3441 		 * If @prev is runnable & has slice left, it has priority and
3442 		 * fetching more just increases latency for the fetched tasks.
3443 		 * Tell pick_task_scx() to keep running @prev. If the BPF
3444 		 * scheduler wants to handle this explicitly, it should
3445 		 * implement ->cpu_release().
3446 		 */
3447 		if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) {
3448 			verdict = SCX_DSP_PREV;
3449 			goto has_tasks;
3450 		}
3451 	}
3452 
3453 	/* if there already are tasks to run, nothing to do */
3454 	if (rq->scx.local_dsq.nr) {
3455 		verdict = SCX_DSP_LOCAL;
3456 		goto has_tasks;
3457 	}
3458 
3459 	verdict = scx_dispatch_sched(root_sch, rq, prev, false);
3460 
3461 	/* scx_dispatch_sched() may have dropped the rq lock */
3462 	if (scx_dsp_prev_stale(rq, prev))
3463 		verdict = SCX_DSP_RETRY;
3464 	if (verdict != SCX_DSP_NONE)
3465 		goto has_tasks;
3466 
3467 	/*
3468 	 * Didn't find another task to run. Keep running @prev unless its own
3469 	 * scheduler set %SCX_OPS_ENQ_LAST and takes the enqueue instead, see
3470 	 * put_prev_task_scx(). Read the scheduler here as the dispatch above
3471 	 * may have dropped the rq lock while @prev changed class or scheduler.
3472 	 */
3473 	if (prev->scx.flags & SCX_TASK_QUEUED) {
3474 		struct scx_sched *prev_sch = scx_task_sched(prev);
3475 
3476 		if ((!(prev_sch->ops.flags & SCX_OPS_ENQ_LAST) ||
3477 		     scx_bypassing(prev_sch, cpu)) && scx_task_can_stay_on_cpu(rq, prev)) {
3478 			__scx_add_event(prev_sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
3479 			verdict = SCX_DSP_PREV;
3480 			goto has_tasks;
3481 		}
3482 
3483 		/*
3484 		 * The scheduler preempts a rescuee like any other task by
3485 		 * zeroing its slice. The grant survives the preemption. The
3486 		 * rescuee gets the unserved remainder back and runs again
3487 		 * behind whatever the scheduler wanted to run first.
3488 		 *
3489 		 * Two sites restore the remainder. When another task won the
3490 		 * pick, put_prev_task_scx() restores it and queues the rescuee
3491 		 * at the tail of the local DSQ.
3492 		 *
3493 		 * When nothing else is runnable, the cpu must not go idle with
3494 		 * the rescuee queued. The keep above takes a rescuee whose
3495 		 * scheduler holds %SCX_CAP_BASE here and doesn't take the last
3496 		 * task through %SCX_OPS_ENQ_LAST.
3497 		 *
3498 		 * The pick then ends its zeroed slice, and the rescue with it,
3499 		 * and refills a default slice. The task continues as an
3500 		 * ordinary one, which is fine as its scheduler holds BASE here.
3501 		 *
3502 		 * For any other rescuee, a reinsert from put_prev_task_scx()
3503 		 * would come after the pick settled on idle. Keep it running on
3504 		 * the remainder here.
3505 		 */
3506 		if (unlikely(prev == scx_rescuee(rq)) && scx_rescue_keep(rq, prev)) {
3507 			verdict = SCX_DSP_PREV;
3508 			goto has_tasks;
3509 		}
3510 	}
3511 	rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
3512 	return SCX_DSP_NONE;
3513 
3514 has_tasks:
3515 	/*
3516 	 * @rq may have extra IMMED tasks without reenq scheduled:
3517 	 *
3518 	 * - rq_is_open() can't reliably tell when and how slice is going to be
3519 	 *   modified for $curr and allows IMMED tasks to be queued while
3520 	 *   dispatch is in progress.
3521 	 *
3522 	 * - A non-IMMED HEAD task can get queued in front of an IMMED task
3523 	 *   between the IMMED queueing and the subsequent scheduling event.
3524 	 *
3525 	 * - When dispatch keeps @prev, even a lone IMMED task queued during the
3526 	 *   dispatch can't run right away.
3527 	 */
3528 	if (unlikely((rq->scx.local_dsq.nr > 1 || verdict == SCX_DSP_PREV) &&
3529 		     rq->scx.nr_immed))
3530 		scx_schedule_reenq_local(rq, 0);
3531 
3532 	rq->scx.flags &= ~SCX_RQ_IN_DISPATCH;
3533 	return verdict;
3534 }
3535 
3536 static void scx_start_task_running(struct rq *rq, struct task_struct *p)
3537 {
3538 	struct scx_sched *sch = scx_task_sched(p);
3539 
3540 	if (p->scx.flags & SCX_TASK_RUN_TRACKED)
3541 		return;
3542 
3543 	scx_cid_sched_update(rq, sch);
3544 
3545 	if (SCX_HAS_OP(sch, running))
3546 		SCX_CALL_OP_TASK(sch, running, rq, p);
3547 
3548 	p->scx.flags |= SCX_TASK_RUN_TRACKED;
3549 }
3550 
3551 static void kick_sync_wait_bal_cb(struct rq *rq);
3552 
3553 static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type)
3554 {
3555 	bool first = type == SNT_PICK;
3556 	bool can_stop_tick;
3557 
3558 	/*
3559 	 * A blocked pick is provisional until proxy resolution completes, see
3560 	 * put_prev_task_scx(). A retry can repick the same donor, so
3561 	 * SNT_REPICK must set it again.
3562 	 */
3563 	rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
3564 	if (sched_proxy_exec() && p->is_blocked &&
3565 	    (type == SNT_PICK || type == SNT_REPICK))
3566 		rq->scx.flags |= SCX_RQ_PROXY_PENDING;
3567 
3568 	/*
3569 	 * Defer the kick_sync wait to a balance callback which can drop rq lock
3570 	 * and enable IRQs. Waiting directly in the pick path would deadlock
3571 	 * against CPUs sending us IPIs (e.g., TLB flushes) while we wait for
3572 	 * them.
3573 	 *
3574 	 * Queue it only once the pick is final, so that the rq lock can't be
3575 	 * released with the callback queued (e.g., by another dispatch during
3576 	 * a core-wide selection). If no SCX task gets picked, there's nothing
3577 	 * to wait for and kick_sync_pending stays set for the next SCX pick.
3578 	 */
3579 	if (unlikely(rq->scx.kick_sync_pending) && (type == SNT_PICK || type == SNT_REPICK))
3580 		queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, kick_sync_wait_bal_cb);
3581 
3582 	if (type == SNT_REPICK)
3583 		return;
3584 
3585 	if (p->scx.flags & SCX_TASK_QUEUED) {
3586 		/*
3587 		 * Core-sched might decide to execute @p before it is
3588 		 * dispatched. Call ops_dequeue() to notify the BPF scheduler.
3589 		 */
3590 		ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC);
3591 		scx_dispatch_dequeue(rq, p);
3592 		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
3593 	}
3594 
3595 	p->se.exec_start = rq_clock_task(rq);
3596 
3597 	/*
3598 	 * See dequeue_task_scx() for why we skip when !QUEUED.
3599 	 *
3600 	 * During a normal switch (@first), a blocked task is only a provisional
3601 	 * donor. Proxy resolution may fail or migrate the donor to another CPU,
3602 	 * so defer ops.running() until scx_proxy_donor_start() confirms that
3603 	 * resolution succeeded.
3604 	 *
3605 	 * !@first denotes restoration after a SAVE/RESTORE cycle. The matching
3606 	 * dequeue already issued ops.stopping(), so restart the session here
3607 	 * regardless of the donor state.
3608 	 */
3609 	if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first))
3610 		scx_start_task_running(rq, p);
3611 
3612 	clr_task_runnable(p, true);
3613 
3614 	/* apply any pending out-of-band slice request before the tick decision */
3615 	apply_task_slice_oob(rq, p);
3616 	can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked;
3617 
3618 	/*
3619 	 * @p is getting newly scheduled or got kicked after someone updated its
3620 	 * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
3621 	 * stopped. See scx_can_stop_tick().
3622 	 *
3623 	 * Moreover, refresh the load_avgs just when transitioning in and out of
3624 	 * nohz. In the future, we might want to add a mechanism to update
3625 	 * load_avgs periodically on tick-stopped CPUs.
3626 	 */
3627 	if (can_stop_tick) {
3628 		if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
3629 			/*
3630 			 * Bypass mode always assigns finite slices, so @p
3631 			 * can't have an infinite slice while bypassing.
3632 			 * Therefore, sched_update_tick_dependency() can safely
3633 			 * evaluate the outgoing task.
3634 			 */
3635 			rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
3636 			sched_update_tick_dependency(rq);
3637 
3638 			update_other_load_avgs(rq);
3639 		}
3640 	} else {
3641 		if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
3642 			rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
3643 			update_other_load_avgs(rq);
3644 		}
3645 
3646 		/*
3647 		 * @rq still references the outgoing scheduling context. A finite
3648 		 * slice or a blocked proxy donor is sufficient by itself to require
3649 		 * the tick.
3650 		 */
3651 		if (tick_nohz_full_cpu(cpu_of(rq)))
3652 			tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
3653 	}
3654 }
3655 
3656 void scx_proxy_donor_start(struct rq *rq)
3657 {
3658 	struct task_struct *donor = rq->donor;
3659 
3660 	lockdep_assert_rq_held(rq);
3661 	rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
3662 
3663 	if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED))
3664 		scx_start_task_running(rq, donor);
3665 }
3666 
3667 static enum scx_cpu_preempt_reason
3668 preempt_reason_from_class(const struct sched_class *class)
3669 {
3670 	if (class == &stop_sched_class)
3671 		return SCX_CPU_PREEMPT_STOP;
3672 	if (class == &dl_sched_class)
3673 		return SCX_CPU_PREEMPT_DL;
3674 	if (class == &rt_sched_class)
3675 		return SCX_CPU_PREEMPT_RT;
3676 	return SCX_CPU_PREEMPT_UNKNOWN;
3677 }
3678 
3679 static void switch_class(struct rq *rq, struct task_struct *next)
3680 {
3681 	struct scx_sched *sch = scx_root_protected_live();
3682 	const struct sched_class *next_class = next->sched_class;
3683 
3684 	if (!(sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT))
3685 		return;
3686 
3687 	/*
3688 	 * The callback is conceptually meant to convey that the CPU is no
3689 	 * longer under the control of SCX. Therefore, don't invoke the callback
3690 	 * if the next class is below SCX (in which case the BPF scheduler has
3691 	 * actively decided not to schedule any tasks on the CPU).
3692 	 */
3693 	if (sched_class_above(&ext_sched_class, next_class))
3694 		return;
3695 
3696 	/*
3697 	 * At this point we know that SCX was preempted by a higher priority
3698 	 * sched_class, so invoke the ->cpu_release() callback if we have not
3699 	 * done so already. We only send the callback once between SCX being
3700 	 * preempted, and it regaining control of the CPU.
3701 	 *
3702 	 * ->cpu_release() complements ->cpu_acquire(), which is emitted the
3703 	 *  next time that dispatch_one() is invoked.
3704 	 */
3705 	if (!rq->scx.cpu_released) {
3706 		if (sch->ops.cpu_release) {
3707 			struct scx_cpu_release_args args = {
3708 				.reason = preempt_reason_from_class(next_class),
3709 				.task = next,
3710 			};
3711 
3712 			SCX_CALL_OP(sch, cpu_release, rq, cpu_of(rq), &args);
3713 		}
3714 		rq->scx.cpu_released = true;
3715 	}
3716 }
3717 
3718 static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
3719 			      struct task_struct *next)
3720 {
3721 	struct scx_sched *sch = scx_task_sched(p);
3722 	bool proxy_put = p->is_blocked && next == rq->idle &&
3723 			 (rq->scx.flags & SCX_RQ_PROXY_PENDING);
3724 	bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING;
3725 	bool rescue_keep = false;
3726 	bool going_idle = !proxy_put && next &&
3727 			  sched_class_above(&ext_sched_class, next->sched_class);
3728 
3729 	rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
3730 
3731 	/* see kick_sync_wait_bal_cb() */
3732 	smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3733 
3734 	update_curr_scx(rq);
3735 
3736 	/*
3737 	 * Preserve the running session when proxy execution refreshes the same
3738 	 * donor around an execution-context switch on this rq.
3739 	 */
3740 	if (next != p && (p->scx.flags & SCX_TASK_QUEUED) &&
3741 	    (p->scx.flags & SCX_TASK_RUN_TRACKED)) {
3742 		if (SCX_HAS_OP(sch, stopping))
3743 			SCX_CALL_OP_TASK(sch, stopping, rq, p, !proxy_block);
3744 
3745 		p->scx.flags &= ~SCX_TASK_RUN_TRACKED;
3746 	}
3747 
3748 	/*
3749 	 * If the slice is consumed, protection ends with it.
3750 	 *
3751 	 * A rescuee keeps going when a preemption or ops.stopping() zeroed its
3752 	 * slice. Here another task won the pick. scx_rescue_keep() restores the
3753 	 * unserved remainder and the reinsert below queues the rescuee at the
3754 	 * tail of the local DSQ. When nothing else is runnable, dispatch_one()
3755 	 * keeps the rescuee running on the remainder instead.
3756 	 */
3757 	if (!p->scx.slice) {
3758 		if (unlikely(p == scx_rescuee(rq)))
3759 			rescue_keep = scx_rescue_keep(rq, p);
3760 		if (!rescue_keep)
3761 			scx_task_slice_ended(rq, p);
3762 	}
3763 
3764 	if (p->scx.flags & SCX_TASK_QUEUED) {
3765 		if (proxy_block)
3766 			goto switch_class;
3767 
3768 		set_task_runnable(rq, p);
3769 
3770 		/*
3771 		 * If the donor is the same and only the mutex owner changes,
3772 		 * avoid triggering another ops.enqueue(): the BPF scheduler has
3773 		 * already admitted the donor, so it can continue running.
3774 		 */
3775 		if (p->is_blocked && next == p)
3776 			goto switch_class;
3777 
3778 		/*
3779 		 * If @p has slice left and is being put, @p is getting
3780 		 * preempted by a higher priority scheduler class or core-sched
3781 		 * forcing a different task. Leave it at the head of the local
3782 		 * DSQ unless it was an IMMED task. IMMED tasks should not
3783 		 * linger on a busy CPU, reenqueue them to the BPF scheduler.
3784 		 *
3785 		 * The exception is an IMMED donor put by proxy_resched_idle():
3786 		 * the CPU isn't busy, proxy resolution is only retrying through
3787 		 * idle, so keep the donor where it can be picked again.
3788 		 *
3789 		 * A task with a kernel slice stays on the local DSQ even while
3790 		 * @sch bypasses. Under bypass, rescue runs @p when the host
3791 		 * lacks %SCX_CAP_BASE, see bypass_enq_target_dsq().
3792 		 */
3793 		if (p->scx.slice && (!scx_bypassing(sch, cpu_of(rq)) ||
3794 				     scx_task_has_kernel_slice(p))) {
3795 			if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) {
3796 				p->scx.flags |= SCX_TASK_REENQ_PREEMPTED;
3797 				scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
3798 			} else {
3799 				u64 enq_flags = 0;
3800 
3801 				/*
3802 				 * The enqueue caps gate the scheduler's
3803 				 * placement. The kernel placed a kernel-slice
3804 				 * task here itself, so reinsert it without a
3805 				 * cap check, at the head like any preempted
3806 				 * task. When scx_rescue_keep() restored a slice
3807 				 * the scheduler zeroed, the scheduler wanted
3808 				 * something else to run first, so append.
3809 				 */
3810 				if (unlikely(scx_task_has_kernel_slice(p))) {
3811 					enq_flags |= SCX_ENQ_IGNORE_CAPS;
3812 					if (!rescue_keep)
3813 						enq_flags |= SCX_ENQ_HEAD;
3814 				} else {
3815 					enq_flags |= SCX_ENQ_HEAD;
3816 					/* only require SCX_CAP_ENQ_IMMED, see scx_caps_for_enq() */
3817 					if (proxy_put && (p->scx.flags & SCX_TASK_IMMED))
3818 						enq_flags |= SCX_ENQ_IMMED;
3819 				}
3820 
3821 				scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
3822 						     enq_flags);
3823 			}
3824 			goto switch_class;
3825 		}
3826 
3827 		/*
3828 		 * If @p is runnable but we're about to enter a lower
3829 		 * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
3830 		 * ops.enqueue() that @p is the only one available for this cpu.
3831 		 * This doesn't apply if the baseline access on the CPU is lost or
3832 		 * proxy resolution temporarily switches to idle.
3833 		 *
3834 		 * Under core scheduling, a pick dispatches only when nothing is
3835 		 * locally runnable and can legitimately go idle with @p still
3836 		 * runnable (see do_pick_task_scx()).
3837 		 */
3838 		if (going_idle && scx_task_can_stay_on_cpu(rq, p)) {
3839 			WARN_ON_ONCE(!sched_core_enabled(rq) &&
3840 				     !(sch->ops.flags & SCX_OPS_ENQ_LAST));
3841 			scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
3842 		} else {
3843 			scx_do_enqueue_task(rq, p, 0, -1);
3844 		}
3845 
3846 		/*
3847 		 * @p may be on the local DSQ now. With %SCX_ENQ_LAST the
3848 		 * scheduler can put the last runnable task there. The plain
3849 		 * insert runs when @p cannot stay on this cpu, and rescue can
3850 		 * still admit it here.
3851 		 *
3852 		 * The cpu must not go idle with @p queued on its local DSQ. The
3853 		 * usual resched doesn't work: the pick has settled on @next and
3854 		 * @p is still curr, so resched_curr() would set need_resched on
3855 		 * @p, which __schedule() clears right after the pick. Set it on
3856 		 * @next instead.
3857 		 */
3858 		if (going_idle && p->scx.dsq == &rq->scx.local_dsq)
3859 			set_tsk_need_resched(next);
3860 	}
3861 
3862 switch_class:
3863 	if (next && next->sched_class != &ext_sched_class) {
3864 		scx_cid_sched_update(rq, NULL);
3865 		switch_class(rq, next);
3866 	}
3867 }
3868 
3869 static void kick_sync_wait_bal_cb(struct rq *rq)
3870 {
3871 	struct scx_kick_syncs __rcu *ks;
3872 	unsigned long *ksyncs;
3873 	bool waited;
3874 	s32 cpu;
3875 
3876 	/*
3877 	 * This callback is queued once the pick is final and flushed within
3878 	 * @rq's own scheduling pass. If a lock taker on a foreign CPU ever
3879 	 * flushes it, the snapshots are unrelated to that CPU. Don't get in the
3880 	 * way and leave the wait pending for @rq's next pick.
3881 	 */
3882 	if (unlikely(cpu_of(rq) != smp_processor_id()))
3883 		return;
3884 
3885 	/*
3886 	 * Clear only when the wait actually runs. The callback can be dropped
3887 	 * without running (e.g., proxy exec zaps the balance callbacks and
3888 	 * picks again when find_proxy_task() fails), in which case the next
3889 	 * pick must queue it again.
3890 	 */
3891 	rq->scx.kick_sync_pending = false;
3892 
3893 	ks = __this_cpu_read(scx_kick_syncs);
3894 	ksyncs = rcu_dereference_sched(ks)->syncs;
3895 
3896 	/*
3897 	 * Drop rq lock and enable IRQs while waiting. IRQs must be enabled
3898 	 * — a target CPU may be waiting for us to process an IPI (e.g. TLB
3899 	 * flush) while we wait for its kick_sync to advance.
3900 	 *
3901 	 * Also, keep advancing our own kick_sync so that new kick_sync waits
3902 	 * targeting us, which can start after we drop the lock, cannot form
3903 	 * cyclic dependencies.
3904 	 */
3905 retry:
3906 	waited = false;
3907 	for_each_cpu(cpu, rq->scx.cpus_to_sync) {
3908 		/*
3909 		 * smp_load_acquire() pairs with smp_store_release() on
3910 		 * kick_sync updates on the target CPUs.
3911 		 */
3912 		if (cpu == cpu_of(rq) ||
3913 		    smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) {
3914 			cpumask_clear_cpu(cpu, rq->scx.cpus_to_sync);
3915 			continue;
3916 		}
3917 
3918 		scx_rq_lock_drop(rq);
3919 		raw_spin_rq_unlock_irq(rq);
3920 		while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) {
3921 			smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
3922 			cpu_relax();
3923 		}
3924 		raw_spin_rq_lock_irq(rq);
3925 		waited = true;
3926 	}
3927 
3928 	if (waited)
3929 		goto retry;
3930 }
3931 
3932 static struct task_struct *first_local_task(struct rq *rq)
3933 {
3934 	return list_first_entry_or_null(&rq->scx.local_dsq.list,
3935 					struct task_struct, scx.dsq_list.node);
3936 }
3937 
3938 /*
3939  * Run dispatch for a pick. The follow-up work is queued by do_pick_task_scx()
3940  * once the pick is final.
3941  */
3942 static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf,
3943 					  struct task_struct *prev)
3944 {
3945 	enum scx_dsp_verdict verdict;
3946 
3947 	rq_unpin_lock(rq, rf);
3948 	verdict = dispatch_one(rq, prev);
3949 	rq_repin_lock(rq, rf);
3950 
3951 	return verdict;
3952 }
3953 
3954 #ifdef CONFIG_SCHED_CORE
3955 /*
3956  * Dispatch for a pick when core scheduling is enabled. The selection picks for
3957  * all SMT siblings and the rq_i->core_pick state it builds must stay atomic
3958  * throughout. If the dispatch released the rq lock, anything can have happened
3959  * in between - return %SCX_DSP_RETRY to restart the selection against current
3960  * state.
3961  */
3962 static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf,
3963 					       struct task_struct *prev)
3964 {
3965 	enum scx_dsp_verdict verdict;
3966 	u32 seq = rq->scx.lock_drop_seq;
3967 
3968 	/* another dispatch is in flight on @rq, let that handle it */
3969 	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
3970 		return SCX_DSP_NONE;
3971 
3972 	rq_unpin_lock(rq, rf);
3973 
3974 	verdict = dispatch_one(rq, prev);
3975 
3976 	/*
3977 	 * A pick that returns a task isn't final under core scheduling: the
3978 	 * selection goes on to pick the siblings, and their dispatch can
3979 	 * release the rq lock, which can't happen with balance callbacks
3980 	 * queued. Don't queue the deferred work as a balance callback, for this
3981 	 * CPU or a sibling, and run it from irq_work instead.
3982 	 *
3983 	 * do_pick_task_scx() leaves %SCX_RQ_BAL_CB_PENDING alone under core
3984 	 * scheduling, so that the early return above leaves it to the dispatch
3985 	 * in flight, which handles it here.
3986 	 */
3987 	if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) {
3988 		rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
3989 		schedule_deferred(rq);
3990 	}
3991 
3992 	rq_repin_lock(rq, rf);
3993 
3994 	/* if dispatch_one() released the rq lock, restart the selection */
3995 	if (rq->scx.lock_drop_seq != seq)
3996 		return SCX_DSP_RETRY;
3997 
3998 	return verdict;
3999 }
4000 #else	/* CONFIG_SCHED_CORE */
4001 static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *rf,
4002 					       struct task_struct *prev)
4003 {
4004 	return SCX_DSP_NONE;
4005 }
4006 #endif	/* CONFIG_SCHED_CORE */
4007 
4008 static struct task_struct *
4009 do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
4010 {
4011 	struct task_struct *prev = rq->donor;
4012 	bool core_sched = sched_core_enabled(rq);
4013 	enum scx_dsp_verdict verdict;
4014 	struct task_struct *p;
4015 
4016 	/* A retry can abandon a provisional blocked-donor pick. */
4017 	rq->scx.flags &= ~SCX_RQ_PROXY_PENDING;
4018 
4019 	/* see kick_sync_wait_bal_cb() */
4020 	smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
4021 
4022 	rq_modified_begin(rq, &ext_sched_class);
4023 
4024 	/*
4025 	 * Core scheduling can get enabled while dispatch_pick() has the rq lock
4026 	 * released. Stick to @core_sched, so that the follow-up work below
4027 	 * matches the dispatch that ran.
4028 	 */
4029 	if (core_sched)
4030 		verdict = dispatch_core_pick(rq, rf, prev);
4031 	else
4032 		verdict = dispatch_pick(rq, rf, prev);
4033 
4034 	if (verdict == SCX_DSP_RETRY) {
4035 		p = RETRY_TASK;
4036 		goto out;
4037 	}
4038 
4039 	/*
4040 	 * If any higher-priority sched class enqueued a runnable task on this
4041 	 * rq during dispatch_one(), abort and return RETRY_TASK, so that the
4042 	 * scheduler loop can restart.
4043 	 *
4044 	 * If @force_scx is true, always try to pick a SCHED_EXT task,
4045 	 * regardless of any higher-priority sched classes activity.
4046 	 */
4047 	if (!force_scx && rq_modified_above(rq, &ext_sched_class)) {
4048 		p = RETRY_TASK;
4049 		goto out;
4050 	}
4051 
4052 	/*
4053 	 * If we're keeping @prev, replenish slice if necessary and keep running
4054 	 * @prev. Otherwise, pop the first one from the local DSQ.
4055 	 */
4056 	if (verdict == SCX_DSP_PREV) {
4057 		p = prev;
4058 		if (!p->scx.slice) {
4059 			/* the slice is consumed, protection ends */
4060 			scx_task_slice_ended(rq, p);
4061 			refill_task_slice_dfl(scx_task_sched(p), p);
4062 		}
4063 	} else {
4064 		p = first_local_task(rq);
4065 		if (!p)
4066 			goto out;
4067 
4068 		if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) {
4069 			struct scx_sched *sch = scx_task_sched(p);
4070 
4071 			if (!scx_bypassing(sch, cpu_of(rq)) &&
4072 			    !sch->warned_zero_slice) {
4073 				printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in %s()\n",
4074 						p->comm, p->pid, __func__);
4075 				sch->warned_zero_slice = true;
4076 			}
4077 			refill_task_slice_dfl(sch, p);
4078 		}
4079 	}
4080 
4081 out:
4082 	if (cpu_of(rq) != smp_processor_id())
4083 		return p;
4084 
4085 	/*
4086 	 * Balance callbacks must be consumed before the rq lock is released, so
4087 	 * queue them only when the pick is final. On RETRY_TASK, or after a
4088 	 * NULL pick from the dl_server, the core continues picking and can
4089 	 * release the rq lock (e.g., prev_balance() or another dispatch). In
4090 	 * that case, run the deferred work from irq_work, which doesn't depend
4091 	 * on the current rq lock section.
4092 	 */
4093 	if (p != RETRY_TASK && (p || !force_scx)) {
4094 		/* under core scheduling, see dispatch_core_pick() */
4095 		if (!core_sched)
4096 			maybe_queue_balance_callback(rq);
4097 	} else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) {
4098 		rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
4099 		schedule_deferred(rq);
4100 	}
4101 
4102 	return p;
4103 }
4104 
4105 static struct task_struct *pick_task_scx(struct rq *rq, struct rq_flags *rf)
4106 {
4107 	return do_pick_task_scx(rq, rf, false);
4108 }
4109 
4110 /*
4111  * Select the next task to run from the ext scheduling class.
4112  *
4113  * Use do_pick_task_scx() directly with @force_scx enabled, since the
4114  * dl_server must always select a sched_ext task.
4115  */
4116 static struct task_struct *
4117 ext_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf)
4118 {
4119 	if (!scx_enabled())
4120 		return NULL;
4121 
4122 	return do_pick_task_scx(dl_se->rq, rf, true);
4123 }
4124 
4125 /*
4126  * Initialize the ext server deadline entity.
4127  */
4128 void ext_server_init(struct rq *rq)
4129 {
4130 	struct sched_dl_entity *dl_se = &rq->ext_server;
4131 
4132 	init_dl_entity(dl_se);
4133 
4134 	dl_server_init(dl_se, rq, ext_server_pick_task);
4135 }
4136 
4137 #ifdef CONFIG_SCHED_CORE
4138 /**
4139  * scx_prio_less - Task ordering for core-sched
4140  * @a: task A
4141  * @b: task B
4142  * @in_fi: in forced idle state
4143  *
4144  * Core-sched is implemented as an additional scheduling layer on top of the
4145  * usual sched_class'es and needs to find out the expected task ordering. For
4146  * SCX, core-sched calls this function to interrogate the task ordering.
4147  *
4148  * A pair of tasks owned by one scheduler is ordered by the owner's
4149  * ops.core_sched_before(). A pair spanning two schedulers is ordered by their
4150  * nearest common ancestor which implements the op - the one case where the op
4151  * is called on tasks that the scheduler delegated to its sub-schedulers and may
4152  * not be scheduling anymore.
4153  *
4154  * When neither applies, or the deciding scheduler is bypassing on either task's
4155  * CPU, the default ordering runs the task which has been waiting longer first.
4156  * A running task counts as the most recently serviced and orders after every
4157  * waiting task. Waiting tasks are compared by @p->scx.runnable_at.
4158  *
4159  * Return: %true if @a should run after @b.
4160  */
4161 bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
4162 		   bool in_fi)
4163 {
4164 	struct scx_sched *sch_a = scx_task_sched(a);
4165 	struct scx_sched *sch_b = scx_task_sched(b);
4166 	struct scx_sched *sch = NULL;
4167 	bool a_running, b_running;
4168 
4169 	if (sch_a == sch_b) {
4170 		if (SCX_HAS_OP(sch_a, core_sched_before))
4171 			sch = sch_a;
4172 	} else {
4173 		s32 level;
4174 
4175 		for (level = min(sch_a->level, sch_b->level); level >= 0; level--) {
4176 			struct scx_sched *anc = sch_a->ancestors[level];
4177 
4178 			if (anc == sch_b->ancestors[level] &&
4179 			    SCX_HAS_OP(anc, core_sched_before)) {
4180 				sch = anc;
4181 				break;
4182 			}
4183 		}
4184 	}
4185 
4186 	/*
4187 	 * scx_prio_less() returns whether @a should run after @b while
4188 	 * ops.core_sched_before() returns whether its first argument should run
4189 	 * before the second. Swap the arguments.
4190 	 *
4191 	 * The const qualifiers are dropped from task_struct pointers when
4192 	 * calling ops.core_sched_before(). Accesses are controlled by the
4193 	 * verifier.
4194 	 */
4195 	if (sch && !scx_bypassing(sch, task_cpu(a)) && !scx_bypassing(sch, task_cpu(b)))
4196 		return SCX_CALL_OP_2TASKS_RET(sch, core_sched_before, task_rq(a),
4197 					      (struct task_struct *)b,
4198 					      (struct task_struct *)a);
4199 
4200 	/*
4201 	 * runnable_at is refreshed only on enqueue, so a task which keeps
4202 	 * occupying its CPU carries a stale stamp. A running task is the most
4203 	 * recently serviced whatever its stamp says. Order it after every
4204 	 * waiting task.
4205 	 */
4206 	a_running = a->on_cpu;
4207 	b_running = b->on_cpu;
4208 	if (a_running != b_running)
4209 		return a_running;
4210 
4211 	return time_after(a->scx.runnable_at, b->scx.runnable_at);
4212 }
4213 #endif	/* CONFIG_SCHED_CORE */
4214 
4215 static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)
4216 {
4217 	struct scx_sched *sch = scx_task_sched(p);
4218 	bool bypassing;
4219 
4220 	/*
4221 	 * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it
4222 	 * can be a good migration opportunity with low cache and memory
4223 	 * footprint. Returning a CPU different than @prev_cpu triggers
4224 	 * immediate rq migration. However, for SCX, as the current rq
4225 	 * association doesn't dictate where the task is going to run, this
4226 	 * doesn't fit well. If necessary, we can later add a dedicated method
4227 	 * which can decide to preempt self to force it through the regular
4228 	 * scheduling path.
4229 	 */
4230 	if (unlikely(wake_flags & WF_EXEC))
4231 		return prev_cpu;
4232 
4233 	bypassing = scx_bypassing(sch, task_cpu(p));
4234 	if (likely(SCX_HAS_OP(sch, select_cpu)) && !bypassing) {
4235 		s32 cpu;
4236 		struct task_struct **ddsp_taskp;
4237 
4238 		ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
4239 		WARN_ON_ONCE(*ddsp_taskp);
4240 		*ddsp_taskp = p;
4241 
4242 		this_rq()->scx.in_select_cpu = true;
4243 		cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p,
4244 					   scx_cpu_arg(prev_cpu), wake_flags);
4245 		cpu = scx_cpu_ret(sch, cpu);
4246 		this_rq()->scx.in_select_cpu = false;
4247 		p->scx.selected_cpu = cpu;
4248 		*ddsp_taskp = NULL;
4249 		if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()"))
4250 			return cpu;
4251 		else
4252 			return prev_cpu;
4253 	} else {
4254 		s32 cpu;
4255 
4256 		/*
4257 		 * While bypassing, the enqueue path routes @p to a bypass DSQ
4258 		 * without consulting the direct-dispatch target, making the
4259 		 * default selection pointless. It doesn't work anyway when the
4260 		 * scheduler does its own idle tracking and the built-in idle
4261 		 * cpumasks are not updated. Leave @p on @prev_cpu.
4262 		 */
4263 		if (bypassing) {
4264 			__scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1);
4265 			p->scx.selected_cpu = prev_cpu;
4266 			return prev_cpu;
4267 		}
4268 
4269 		cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0);
4270 		if (cpu >= 0) {
4271 			/*
4272 			 * Carry the slice refill and let the insertion commit
4273 			 * it under rq lock. See the write rules.
4274 			 */
4275 			__scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
4276 			p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl);
4277 			p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL;
4278 			p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;
4279 		} else {
4280 			cpu = prev_cpu;
4281 		}
4282 		p->scx.selected_cpu = cpu;
4283 
4284 		return cpu;
4285 	}
4286 }
4287 
4288 static void task_woken_scx(struct rq *rq, struct task_struct *p)
4289 {
4290 	run_deferred(rq);
4291 }
4292 
4293 static void set_cpus_allowed_scx(struct task_struct *p,
4294 				 struct affinity_context *ac)
4295 {
4296 	struct scx_sched *sch = scx_task_sched(p);
4297 
4298 	set_cpus_allowed_common(p, ac);
4299 
4300 	if (task_dead_and_done(p))
4301 		return;
4302 
4303 	/*
4304 	 * The effective cpumask is stored in @p->cpus_ptr which may temporarily
4305 	 * differ from the configured one in @p->cpus_mask. Always tell the bpf
4306 	 * scheduler the effective one.
4307 	 *
4308 	 * Fine-grained memory write control is enforced by BPF making the const
4309 	 * designation pointless. Cast it away when calling the operation.
4310 	 *
4311 	 * The cid form receives the initial mask when the task is enabled and
4312 	 * hears about changes only afterwards, see struct scx_enable_args.
4313 	 */
4314 	if (SCX_HAS_OP(sch, set_cpumask) &&
4315 	    (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED))
4316 		scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
4317 }
4318 
4319 static void drain_offline_rq(struct rq *rq)
4320 {
4321 	struct task_struct *p, *n;
4322 
4323 	scx_rescue_flush(rq);
4324 
4325 	/*
4326 	 * An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
4327 	 * onto the local DSQ so that they run here and balance_push() moves
4328 	 * them off.
4329 	 */
4330 	list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
4331 		if (p->scx.dsq == &rq->scx.local_dsq)
4332 			continue;
4333 		guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK);
4334 	}
4335 }
4336 
4337 static void handle_hotplug(struct rq *rq, bool online)
4338 {
4339 	struct scx_sched *sch = scx_root_protected();
4340 	s32 cpu = cpu_of(rq);
4341 	s32 cpu_or_cid = cpu;
4342 
4343 	if (!online) {
4344 		guard(rq_lock_irqsave)(rq);
4345 		update_rq_clock(rq);
4346 		rq->scx.flags &= ~SCX_RQ_ONLINE;
4347 		drain_offline_rq(rq);
4348 	}
4349 
4350 	atomic_long_inc(&scx_hotplug_seq);
4351 	scx_update_frozen_cpus(cpu, online);
4352 
4353 	/*
4354 	 * scx_root updates are protected by cpus_read_lock() and will stay
4355 	 * stable here. Note that we can't depend on scx_enabled() test as the
4356 	 * hotplug ops need to be enabled before __scx_enabled is set.
4357 	 */
4358 	if (unlikely(!sch)) {
4359 		/* no root to notify, set %SCX_RQ_ONLINE right away */
4360 		if (online) {
4361 			guard(rq_lock_irqsave)(rq);
4362 			rq->scx.flags |= SCX_RQ_ONLINE;
4363 		}
4364 		return;
4365 	}
4366 
4367 	if (scx_enabled())
4368 		scx_idle_update_selcpu_topology(&sch->ops);
4369 
4370 	/*
4371 	 * The tables can't be retired while this function is running as the
4372 	 * retirement is inside cpus_read_lock. However, scx_cpu_arg() is
4373 	 * awkward here as the tables can be NULL after root enable failure and
4374 	 * lockdep would trigger without surrounding rcu_read_lock(). Open code
4375 	 * the translation. If the table is NULL, the ops are also cleared and
4376 	 * @cpu_or_cid goes unused.
4377 	 */
4378 	if (scx_is_cid_type()) {
4379 		s16 *tbl = rcu_dereference_check(scx_cpu_to_cid_tbl,
4380 						 lockdep_is_cpus_held());
4381 
4382 		if (tbl) {
4383 			struct scx_sched *pos;
4384 
4385 			cpu_or_cid = tbl[cpu];
4386 
4387 			guard(raw_spinlock_irqsave)(&scx_sched_lock);
4388 			list_for_each_entry(pos, &scx_sched_all, all) {
4389 				struct scx_cmask *mask = pos->online_cmask;
4390 
4391 				if (mask)
4392 					__assign_bit(cpu_or_cid, (unsigned long *)mask->bits,
4393 						     online);
4394 			}
4395 		}
4396 	}
4397 
4398 	/*
4399 	 * ecaps are delivered for @cpu only after ops.cpu_online() and before
4400 	 * ops.cpu_offline(). The ecaps ops may read online_cmask, so
4401 	 * scx_offline_ecaps() and scx_online_ecaps() run after its update.
4402 	 */
4403 	if (!online)
4404 		scx_offline_ecaps(rq);
4405 
4406 	if (online && SCX_HAS_OP(sch, cpu_online))
4407 		SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid);
4408 	else if (!online && SCX_HAS_OP(sch, cpu_offline))
4409 		SCX_CALL_OP(sch, cpu_offline, NULL, cpu_or_cid);
4410 	else
4411 		scx_exit(sch, SCX_EXIT_UNREG_KERN,
4412 			 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
4413 			 "cpu %d going %s, exiting scheduler", cpu,
4414 			 online ? "online" : "offline");
4415 
4416 	if (online) {
4417 		/*
4418 		 * The cpu must not call ops.enqueue() or ops.dispatch() before
4419 		 * the root's ops.cpu_online(). The core's rq online state can't
4420 		 * gate them, as the sched domain rebuild in
4421 		 * sched_cpu_activate() can put @rq online before this function
4422 		 * runs. Set %SCX_RQ_ONLINE here.
4423 		 *
4424 		 * scx_online_ecaps() queues a reseed whose ecaps ops run in
4425 		 * dispatch context on the cpu and can insert into its local
4426 		 * DSQ. Set the flag first.
4427 		 */
4428 		scoped_guard (rq_lock_irqsave, rq)
4429 			rq->scx.flags |= SCX_RQ_ONLINE;
4430 		scx_online_ecaps(rq);
4431 	}
4432 }
4433 
4434 void scx_rq_activate(struct rq *rq)
4435 {
4436 	handle_hotplug(rq, true);
4437 }
4438 
4439 void scx_rq_deactivate(struct rq *rq)
4440 {
4441 	handle_hotplug(rq, false);
4442 }
4443 
4444 static bool check_rq_for_timeouts(struct rq *rq)
4445 {
4446 	struct scx_sched *sch;
4447 	struct task_struct *p;
4448 	struct rq_flags rf;
4449 	bool timed_out = false;
4450 
4451 	rq_lock_irqsave(rq, &rf);
4452 	sch = rcu_dereference_bh(scx_root);
4453 	if (unlikely(!sch))
4454 		goto out_unlock;
4455 
4456 	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) {
4457 		/*
4458 		 * @p's owner schedules @p, or its host while the owner is
4459 		 * bypassing. Test @p against that sched's timeout and blame it.
4460 		 */
4461 		struct scx_sched *sch = scx_exec_sched(scx_task_sched(p), cpu_of(rq));
4462 		unsigned long last_runnable = p->scx.runnable_at;
4463 
4464 		if (unlikely(time_after(jiffies,
4465 					last_runnable + READ_ONCE(sch->watchdog_timeout)))) {
4466 			u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
4467 
4468 			__scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq),
4469 				   "%s[%d] failed to run for %u.%03us",
4470 				   p->comm, p->pid, dur_ms / 1000,
4471 				   dur_ms % 1000);
4472 			timed_out = true;
4473 			break;
4474 		}
4475 	}
4476 out_unlock:
4477 	rq_unlock_irqrestore(rq, &rf);
4478 	return timed_out;
4479 }
4480 
4481 static void scx_watchdog_workfn(struct work_struct *work)
4482 {
4483 	unsigned long intv;
4484 	int cpu;
4485 
4486 	WRITE_ONCE(scx_watchdog_timestamp, jiffies);
4487 
4488 	for_each_online_cpu(cpu) {
4489 		if (unlikely(check_rq_for_timeouts(cpu_rq(cpu))))
4490 			break;
4491 
4492 		cond_resched();
4493 	}
4494 
4495 	intv = READ_ONCE(scx_watchdog_interval);
4496 	if (intv < ULONG_MAX)
4497 		queue_delayed_work(system_dfl_wq, to_delayed_work(work), intv);
4498 }
4499 
4500 void scx_tick(struct rq *rq)
4501 {
4502 	struct scx_sched *root;
4503 	unsigned long last_check;
4504 
4505 	if (!scx_enabled())
4506 		return;
4507 
4508 	root = rcu_dereference_bh(scx_root);
4509 	if (unlikely(!root))
4510 		return;
4511 
4512 	last_check = READ_ONCE(scx_watchdog_timestamp);
4513 	if (unlikely(time_after(jiffies,
4514 				last_check + READ_ONCE(root->watchdog_timeout)))) {
4515 		u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
4516 
4517 		scx_exit(root, SCX_EXIT_ERROR_STALL, 0,
4518 			 "watchdog failed to check in for %u.%03us",
4519 			 dur_ms / 1000, dur_ms % 1000);
4520 	}
4521 
4522 	update_other_load_avgs(rq);
4523 }
4524 
4525 static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued)
4526 {
4527 	struct scx_sched *sch = scx_task_sched(donor);
4528 
4529 	update_curr_scx(rq);
4530 
4531 	/*
4532 	 * While disabling, always resched as we can't trust the slice
4533 	 * management.
4534 	 */
4535 	if (scx_bypassing(sch, cpu_of(rq)))
4536 		expire_task_slice(donor);
4537 	else if (SCX_HAS_OP(sch, tick))
4538 		SCX_CALL_OP_TASK(sch, tick, rq, donor);
4539 
4540 	if (!donor->scx.slice) {
4541 		/* the slice can't be trusted while bypassing */
4542 		if (READ_ONCE(donor->scx.lazy_resched) &&
4543 		    !scx_bypassing(sch, cpu_of(rq)))
4544 			scx_resched_curr_lazy(rq);
4545 		else
4546 			resched_curr(rq);
4547 	}
4548 }
4549 
4550 #ifdef CONFIG_EXT_GROUP_SCHED
4551 static struct cgroup *tg_cgrp(struct task_group *tg)
4552 {
4553 	/*
4554 	 * If CGROUP_SCHED is disabled, @tg is NULL. If @tg is an autogroup,
4555 	 * @tg->css.cgroup is NULL. In both cases, @tg can be treated as the
4556 	 * root cgroup.
4557 	 */
4558 	if (tg && tg->css.cgroup)
4559 		return tg->css.cgroup;
4560 	else
4561 		return &cgrp_dfl_root.cgrp;
4562 }
4563 
4564 #define SCX_INIT_TASK_ARGS_CGROUP(cgrp)		.cgroup = (cgrp),
4565 
4566 #else	/* CONFIG_EXT_GROUP_SCHED */
4567 
4568 #define SCX_INIT_TASK_ARGS_CGROUP(cgrp)
4569 
4570 #endif	/* CONFIG_EXT_GROUP_SCHED */
4571 
4572 /**
4573  * __scx_init_task - Initialize a task for a sched
4574  * @sch: sched to initialize @p for
4575  * @p: task of interest
4576  * @cgrp: cgroup @p is joining, %NULL for @p's current task_group's cgroup
4577  * @fork: %true if @p is being forked
4578  *
4579  * Pre-commit cgroup migration passes @cgrp explicitly as @p's task_group
4580  * still reflects the source.
4581  *
4582  * Return 0 on success, -errno on failure.
4583  */
4584 int __scx_init_task(struct scx_sched *sch, struct task_struct *p,
4585 		    struct cgroup *cgrp, bool fork)
4586 {
4587 	int ret;
4588 
4589 	p->scx.disallow = false;
4590 
4591 	if (SCX_HAS_OP(sch, init_task)) {
4592 		struct scx_init_task_args args = {
4593 			SCX_INIT_TASK_ARGS_CGROUP(cgrp ?: tg_cgrp(task_group(p)))
4594 			.fork = fork,
4595 		};
4596 
4597 		ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args);
4598 		if (unlikely(ret)) {
4599 			ret = scx_ops_sanitize_err(sch, "init_task", ret);
4600 			return ret;
4601 		}
4602 	}
4603 
4604 	if (p->scx.disallow) {
4605 		if (unlikely(scx_parent(sch))) {
4606 			scx_error(sch, "non-root ops.init_task() set task->scx.disallow for %s[%d]",
4607 				  p->comm, p->pid);
4608 		} else if (unlikely(fork)) {
4609 			scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork",
4610 				  p->comm, p->pid);
4611 		} else if (unlikely(scx_enable_state() != SCX_ENABLING)) {
4612 			scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] outside the enable path",
4613 				  p->comm, p->pid);
4614 		} else {
4615 			struct rq *rq;
4616 			struct rq_flags rf;
4617 
4618 			rq = task_rq_lock(p, &rf);
4619 
4620 			/*
4621 			 * We're in the load path and @p->policy will be applied
4622 			 * right after. Reverting @p->policy here and rejecting
4623 			 * %SCHED_EXT transitions from scx_check_setscheduler()
4624 			 * guarantees that if ops.init_task() sets @p->disallow,
4625 			 * @p can never be in SCX.
4626 			 */
4627 			if (p->policy == SCHED_EXT) {
4628 				p->policy = SCHED_NORMAL;
4629 				atomic_long_inc(&scx_nr_rejected);
4630 			}
4631 
4632 			task_rq_unlock(rq, p, &rf);
4633 		}
4634 	}
4635 
4636 	return 0;
4637 }
4638 
4639 static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)
4640 {
4641 	struct rq *rq = task_rq(p);
4642 	u32 weight;
4643 
4644 	lockdep_assert_rq_held(rq);
4645 
4646 	/*
4647 	 * Verify the task is not in BPF scheduler's custody. If flag
4648 	 * transitions are consistent, the flag should always be clear
4649 	 * here.
4650 	 */
4651 	WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
4652 
4653 	/*
4654 	 * Set the weight before calling ops.enable() so that the scheduler
4655 	 * doesn't see a stale value if they inspect the task struct.
4656 	 */
4657 	if (task_has_idle_policy(p))
4658 		weight = WEIGHT_IDLEPRIO;
4659 	else
4660 		weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
4661 
4662 	p->scx.weight = sched_weight_to_cgroup(weight);
4663 	p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED;
4664 
4665 	if (SCX_HAS_OP(sch, enable)) {
4666 		if (scx_is_cid_type()) {
4667 			struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr);
4668 			struct scx_enable_args args = {
4669 				.cmask_arena_addr = scx_kaddr_to_arena(sch, cmask),
4670 			};
4671 
4672 			SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args);
4673 		} else {
4674 			SCX_CALL_OP_TASK(sch, enable, rq, p);
4675 		}
4676 	}
4677 
4678 	/*
4679 	 * The initial mask also goes out through set_cmask() so a scheduler can
4680 	 * track affinity there alone, and before set_weight() so that the mask
4681 	 * is in place when weight-dependent state is derived, see struct
4682 	 * scx_enable_args.
4683 	 */
4684 	if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
4685 		scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr);
4686 
4687 	if (SCX_HAS_OP(sch, set_weight))
4688 		SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
4689 }
4690 
4691 void scx_enable_task(struct scx_sched *sch, struct task_struct *p)
4692 {
4693 	__scx_enable_task(sch, p);
4694 	scx_set_task_state(p, SCX_TASK_ENABLED);
4695 }
4696 
4697 /**
4698  * scx_reset_task_fields - Reset what @p carries from its sched's control
4699  * @rq: @p's rq, locked
4700  * @p: task leaving its sched's control
4701  *
4702  * The vtime, the slice and what rides on it, the reenqueue count and the
4703  * watchdog wait belong to the sched while it controls @p. The control ends when
4704  * @p is taken off the sched or leaves the ext class while staying on the sched.
4705  * Reset them so that @p starts clean wherever it goes.
4706  */
4707 static void scx_reset_task_fields(struct rq *rq, struct task_struct *p)
4708 {
4709 	lockdep_assert_rq_held(rq);
4710 
4711 	p->scx.dsq_vtime = 0;
4712 	scx_task_slice_ended(rq, p);
4713 	scx_set_task_slice(p, 0);
4714 	p->scx.reenq_cnt = 0;
4715 
4716 	/*
4717 	 * If @p is runnable but not running, its wait started at runnable_at,
4718 	 * which ops_dequeue() didn't reset. Leaving the sched's control ends
4719 	 * the wait. Reset runnable_at the next time @p is enqueued on SCX, so
4720 	 * that the watchdog doesn't count the time @p spends elsewhere.
4721 	 */
4722 	p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
4723 }
4724 
4725 static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
4726 {
4727 	struct rq *rq = task_rq(p);
4728 
4729 	lockdep_assert_rq_held(rq);
4730 	WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED);
4731 
4732 	clear_direct_dispatch(p);
4733 
4734 	if (SCX_HAS_OP(sch, disable))
4735 		SCX_CALL_OP_TASK(sch, disable, rq, p);
4736 	scx_set_task_state(p, SCX_TASK_READY);
4737 
4738 	/*
4739 	 * Verify the task is not in BPF scheduler's custody. If flag
4740 	 * transitions are consistent, the flag should always be clear
4741 	 * here.
4742 	 */
4743 	WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
4744 }
4745 
4746 void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)
4747 {
4748 	struct scx_exit_task_args args = {
4749 		.cancelled = false,
4750 	};
4751 
4752 	lockdep_assert_held(&p->pi_lock);
4753 	lockdep_assert_rq_held(task_rq(p));
4754 
4755 	switch (scx_get_task_state(p)) {
4756 	case SCX_TASK_NONE:
4757 		/*
4758 		 * No op to call, but @p may still carry @sch's state, e.g. a
4759 		 * task punted by scx_fail_parent() and being rescued.
4760 		 */
4761 		scx_reset_task_fields(task_rq(p), p);
4762 		return;
4763 	case SCX_TASK_INIT:
4764 		args.cancelled = true;
4765 		break;
4766 	case SCX_TASK_READY:
4767 		scx_reset_task_fields(task_rq(p), p);
4768 		break;
4769 	case SCX_TASK_ENABLED:
4770 		scx_disable_task(sch, p);
4771 		/* after ops.disable() has seen the final values */
4772 		scx_reset_task_fields(task_rq(p), p);
4773 		break;
4774 	default:
4775 		WARN_ON_ONCE(true);
4776 		return;
4777 	}
4778 
4779 	if (SCX_HAS_OP(sch, exit_task))
4780 		SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
4781 }
4782 
4783 /*
4784  * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never
4785  * ran. The task state has not been transitioned, so this mirrors the
4786  * SCX_TASK_INIT branch in __scx_disable_and_exit_task().
4787  */
4788 void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p)
4789 {
4790 	struct scx_exit_task_args args = { .cancelled = true };
4791 
4792 	lockdep_assert_held(&p->pi_lock);
4793 	lockdep_assert_rq_held(task_rq(p));
4794 
4795 	/* @p was never associated with @sch, dispatch on the explicit @sch */
4796 	if (SCX_HAS_OP(sch, exit_task))
4797 		__SCX_CALL_OP_TASK(sch, ops, exit_task, task_rq(p), p, &args);
4798 }
4799 
4800 void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)
4801 {
4802 	__scx_disable_and_exit_task(sch, p);
4803 
4804 	/*
4805 	 * If set, @p exited between __scx_init_task() and scx_enable_task() in
4806 	 * scx_sub_enable() and is initialized for both the associated sched and
4807 	 * its parent. Exit for the child too - scx_enable_task() never ran for
4808 	 * it, so undo only init_task. The flag is only set on the sub-enable
4809 	 * path, so it's always clear when @p arrives here in %SCX_TASK_NONE.
4810 	 */
4811 	if (p->scx.flags & SCX_TASK_SUB_INIT) {
4812 		if (!WARN_ON_ONCE(!scx_enabling_sub_sched))
4813 			scx_sub_init_cancel_task(scx_enabling_sub_sched, p);
4814 		p->scx.flags &= ~SCX_TASK_SUB_INIT;
4815 	}
4816 
4817 	scx_set_task_sched(p, NULL);
4818 	scx_set_task_state(p, SCX_TASK_NONE);
4819 }
4820 
4821 void init_scx_entity(struct sched_ext_entity *scx)
4822 {
4823 	memset(scx, 0, sizeof(*scx));
4824 	INIT_LIST_HEAD(&scx->dsq_list.node);
4825 	RB_CLEAR_NODE(&scx->dsq_priq);
4826 	scx->sticky_cpu = -1;
4827 	scx->holding_cpu = -1;
4828 	scx->runnable_cpu = -1;
4829 	INIT_LIST_HEAD(&scx->runnable_node);
4830 	scx->runnable_at = jiffies;
4831 	scx->ddsp_dsq_id = SCX_DSQ_INVALID;
4832 	scx->slice = SCX_SLICE_DFL;
4833 }
4834 
4835 /* See scx_tid_alloc / scx_tid_cursor. */
4836 static u64 scx_alloc_tid(void)
4837 {
4838 	struct scx_tid_alloc *ta;
4839 
4840 	guard(preempt)();
4841 	ta = this_cpu_ptr(&scx_tid_alloc);
4842 
4843 	if (unlikely(ta->next >= ta->end)) {
4844 		ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor);
4845 		ta->end = ta->next + SCX_TID_CHUNK;
4846 	}
4847 	return ta->next++;
4848 }
4849 
4850 static void scx_tid_hash_insert(struct task_struct *p)
4851 {
4852 	int ret;
4853 
4854 	lockdep_assert_held(&scx_tasks_lock);
4855 
4856 	ret = rhashtable_lookup_insert_fast(&scx_tid_hash,
4857 					    &p->scx.tid_hash_node,
4858 					    scx_tid_hash_params);
4859 	WARN_ON_ONCE(ret);
4860 }
4861 
4862 void scx_pre_fork(struct task_struct *p)
4863 {
4864 	/*
4865 	 * BPF scheduler enable/disable paths want to be able to iterate and
4866 	 * update all tasks which can become complex when racing forks. As
4867 	 * enable/disable are very cold paths, let's use a percpu_rwsem to
4868 	 * exclude forks.
4869 	 */
4870 	percpu_down_read(&scx_fork_rwsem);
4871 }
4872 
4873 int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs)
4874 {
4875 	s32 ret;
4876 
4877 	percpu_rwsem_assert_held(&scx_fork_rwsem);
4878 
4879 	p->scx.tid = scx_alloc_tid();
4880 
4881 	if (scx_init_task_enabled) {
4882 #ifdef CONFIG_EXT_SUB_SCHED
4883 		struct scx_sched *sch = scx_cgroup_sched(kargs->cset->dfl_cgrp);
4884 #else
4885 		struct scx_sched *sch = scx_root_protected_live();
4886 #endif
4887 		scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
4888 		ret = __scx_init_task(sch, p, NULL, true);
4889 		if (unlikely(ret)) {
4890 			scx_set_task_state(p, SCX_TASK_NONE);
4891 			return ret;
4892 		}
4893 		scx_set_task_state(p, SCX_TASK_INIT);
4894 		scx_set_task_sched(p, sch);
4895 	}
4896 
4897 	return 0;
4898 }
4899 
4900 void scx_post_fork(struct task_struct *p)
4901 {
4902 	if (scx_init_task_enabled) {
4903 		scx_set_task_state(p, SCX_TASK_READY);
4904 
4905 		/*
4906 		 * Enable the task immediately if it's running on sched_ext.
4907 		 * Otherwise, it'll be enabled in switching_to_scx() if and
4908 		 * when it's ever configured to run with a SCHED_EXT policy.
4909 		 */
4910 		if (p->sched_class == &ext_sched_class) {
4911 			struct rq_flags rf;
4912 			struct rq *rq;
4913 
4914 			rq = task_rq_lock(p, &rf);
4915 			scx_enable_task(scx_task_sched(p), p);
4916 			task_rq_unlock(rq, p, &rf);
4917 		}
4918 	}
4919 
4920 	scoped_guard(raw_spinlock_irq, &scx_tasks_lock) {
4921 		list_add_tail(&p->scx.tasks_node, &scx_tasks);
4922 		if (scx_tid_to_task_enabled())
4923 			scx_tid_hash_insert(p);
4924 	}
4925 
4926 	percpu_up_read(&scx_fork_rwsem);
4927 }
4928 
4929 void scx_cancel_fork(struct task_struct *p)
4930 {
4931 	if (scx_init_task_enabled) {
4932 		struct rq *rq;
4933 		struct rq_flags rf;
4934 
4935 		rq = task_rq_lock(p, &rf);
4936 		WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY);
4937 		scx_disable_and_exit_task(scx_task_sched(p), p);
4938 		task_rq_unlock(rq, p, &rf);
4939 	}
4940 
4941 	percpu_up_read(&scx_fork_rwsem);
4942 }
4943 
4944 /**
4945  * task_dead_and_done - Is a task dead and done running?
4946  * @p: target task
4947  *
4948  * Once sched_ext_dead() removes the dead task from scx_tasks and exits it, the
4949  * task no longer exists from SCX's POV. However, certain sched_class ops may be
4950  * invoked on these dead tasks leading to failures - e.g. sched_setscheduler()
4951  * may try to switch a task which finished sched_ext_dead() back into SCX
4952  * triggering invalid SCX task state transitions and worse.
4953  *
4954  * Once a task has finished the final switch, sched_ext_dead() is the only thing
4955  * that needs to happen on the task. Use this test to short-circuit sched_class
4956  * operations which may be called on dead tasks.
4957  */
4958 static bool task_dead_and_done(struct task_struct *p)
4959 {
4960 	struct rq *rq = task_rq(p);
4961 
4962 	lockdep_assert_rq_held(rq);
4963 
4964 	/*
4965 	 * In do_task_dead(), a dying task sets %TASK_DEAD with preemption
4966 	 * disabled and __schedule(). If @p has %TASK_DEAD set and off CPU, @p
4967 	 * won't ever run again.
4968 	 */
4969 	return unlikely(READ_ONCE(p->__state) == TASK_DEAD) &&
4970 		!task_on_cpu(rq, p);
4971 }
4972 
4973 void sched_ext_dead(struct task_struct *p)
4974 {
4975 	/*
4976 	 * By the time control reaches here, @p has %TASK_DEAD set, switched out
4977 	 * for the last time and then dropped the rq lock - task_dead_and_done()
4978 	 * should be returning %true nullifying the straggling sched_class ops.
4979 	 * Remove from scx_tasks and exit @p.
4980 	 */
4981 	scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) {
4982 		list_del_init(&p->scx.tasks_node);
4983 		if (scx_tid_to_task_enabled())
4984 			rhashtable_remove_fast(&scx_tid_hash,
4985 					       &p->scx.tid_hash_node,
4986 					       scx_tid_hash_params);
4987 	}
4988 
4989 	/*
4990 	 * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY ->
4991 	 * ENABLED transitions can't race us. Disable ops for @p.
4992 	 *
4993 	 * %SCX_TASK_DEAD synchronizes against cgroup task iteration - see
4994 	 * scx_task_iter_next_locked(). NONE tasks need no marking: cgroup
4995 	 * iteration is only used from sub-sched paths, which require root
4996 	 * enabled. Root enable transitions every live task to at least READY.
4997 	 *
4998 	 * %INIT_BEGIN means ops.init_task() is running for @p. Don't call
4999 	 * into ops; transition to %DEAD so the post-init recheck unwinds
5000 	 * via scx_sub_init_cancel_task().
5001 	 */
5002 	if (scx_get_task_state(p) != SCX_TASK_NONE) {
5003 		struct rq_flags rf;
5004 		struct rq *rq;
5005 
5006 		rq = task_rq_lock(p, &rf);
5007 		if (scx_get_task_state(p) != SCX_TASK_INIT_BEGIN)
5008 			scx_disable_and_exit_task(scx_task_sched(p), p);
5009 		scx_set_task_state(p, SCX_TASK_DEAD);
5010 		task_rq_unlock(rq, p, &rf);
5011 	}
5012 }
5013 
5014 static void reweight_task_scx(struct rq *rq, struct task_struct *p,
5015 			      const struct load_weight *lw)
5016 {
5017 	struct scx_sched *sch = scx_task_sched(p);
5018 
5019 	lockdep_assert_rq_held(task_rq(p));
5020 
5021 	if (task_dead_and_done(p))
5022 		return;
5023 
5024 	/*
5025 	 * When switching sched_class away from SCX, reweight_task_scx()
5026 	 * is called _after_ scx_disable_task(). Skip calling ops.set_weight()
5027 	 * since the BPF scheduler may have already forgotten the task in
5028 	 * ops.disable().
5029 	 * p->scx.weight will be recalculated in scx_enable_task() if the task
5030 	 * ever returns to SCX class.
5031 	 */
5032 	if (scx_get_task_state(p) != SCX_TASK_ENABLED)
5033 		return;
5034 
5035 	p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
5036 	if (SCX_HAS_OP(sch, set_weight))
5037 		SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
5038 }
5039 
5040 static void prio_changed_scx(struct rq *rq, struct task_struct *p, u64 oldprio)
5041 {
5042 }
5043 
5044 static void switching_to_scx(struct rq *rq, struct task_struct *p)
5045 {
5046 	struct scx_sched *sch = scx_task_sched(p);
5047 
5048 	if (task_dead_and_done(p))
5049 		return;
5050 
5051 	scx_enable_task(sch, p);
5052 
5053 	/*
5054 	 * set_cpus_allowed_scx() is not called while @p is associated with a
5055 	 * different scheduler class. Keep the BPF scheduler up-to-date. The cid
5056 	 * form gets its mask from scx_enable_task().
5057 	 */
5058 	if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
5059 		scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr);
5060 }
5061 
5062 static void switched_from_scx(struct rq *rq, struct task_struct *p)
5063 {
5064 	/*
5065 	 * A class change of the running task: sched_change_begin() put @p with
5066 	 * no successor, which leaves rq->scx.sched set.
5067 	 */
5068 	if (task_current_donor(rq, p))
5069 		scx_cid_sched_update(rq, NULL);
5070 
5071 	if (task_dead_and_done(p))
5072 		return;
5073 
5074 	/*
5075 	 * %NONE means SCX is no longer tracking @p at the task level (e.g.
5076 	 * scx_fail_parent() handed @p back to the parent at NONE pending the
5077 	 * parent's own teardown). There is nothing to disable; calling
5078 	 * scx_disable_task() would WARN on the non-%ENABLED state and trigger a
5079 	 * NONE -> READY validation failure.
5080 	 *
5081 	 * Either way, reset @p after ops.disable() has seen the final values. A
5082 	 * %NONE @p can still be a rescuee under the parent.
5083 	 */
5084 	if (scx_get_task_state(p) != SCX_TASK_NONE)
5085 		scx_disable_task(scx_task_sched(p), p);
5086 	scx_reset_task_fields(rq, p);
5087 }
5088 
5089 static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
5090 
5091 int scx_check_setscheduler(struct task_struct *p, int policy)
5092 {
5093 	lockdep_assert_rq_held(task_rq(p));
5094 
5095 	/* If disallow, reject transitioning into SCX. */
5096 	if (scx_enabled() && READ_ONCE(p->scx.disallow) &&
5097 	    p->policy != policy && policy == SCHED_EXT)
5098 		return -EACCES;
5099 
5100 	return 0;
5101 }
5102 
5103 /*
5104  * Don't carry a donor retained by another class into sched_ext. The caller
5105  * has updated the rq clock and invokes this immediately before
5106  * sched_change_begin() records the task's queued state.
5107  */
5108 void scx_prepare_setscheduler(struct task_struct *p, int policy)
5109 {
5110 	lockdep_assert_held(&p->pi_lock);
5111 	lockdep_assert_rq_held(task_rq(p));
5112 
5113 	if (scx_enabled() && p->policy != policy && policy == SCHED_EXT)
5114 		scx_block_proxy_donor(task_rq(p), p);
5115 }
5116 
5117 static void process_ddsp_deferred_locals(struct rq *rq)
5118 {
5119 	struct task_struct *p;
5120 
5121 	lockdep_assert_rq_held(rq);
5122 
5123 	/*
5124 	 * Now that @rq can be unlocked, execute the deferred enqueueing of
5125 	 * tasks directly dispatched to the local DSQs of other CPUs. See
5126 	 * direct_dispatch(). Keep popping from the head instead of using
5127 	 * list_for_each_entry_safe() as dispatch_local_dsq() may unlock @rq
5128 	 * temporarily.
5129 	 */
5130 	while ((p = list_first_entry_or_null(&rq->scx.ddsp_deferred_locals,
5131 				struct task_struct, scx.dsq_list.node))) {
5132 		struct scx_sched *sch = scx_task_sched(p);
5133 		struct scx_dispatch_q *dsq;
5134 		u64 dsq_id = p->scx.ddsp_dsq_id;
5135 		u64 enq_flags = p->scx.ddsp_enq_flags;
5136 		u64 slice = p->scx.ddsp_slice;
5137 		u64 vtime = p->scx.ddsp_vtime;
5138 
5139 		list_del_init(&p->scx.dsq_list.node);
5140 		clear_direct_dispatch(p);
5141 
5142 		dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
5143 		if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
5144 			dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
5145 	}
5146 }
5147 
5148 /*
5149  * Determine whether @p should be reenqueued from a local DSQ.
5150  *
5151  * @reenq_flags is mutable and accumulates state across the DSQ walk:
5152  *
5153  * - %SCX_REENQ_TSR_NOT_FIRST: Set after the first task is visited. "First"
5154  *   tracks position in the DSQ list, not among IMMED tasks. A non-IMMED task at
5155  *   the head consumes the first slot.
5156  *
5157  * - %SCX_REENQ_TSR_RQ_OPEN: Set by reenq_local() before the walk if
5158  *   rq_is_open() is true.
5159  *
5160  * An IMMED task is kept (returns %false) only if it's the first task in the DSQ
5161  * AND the current task is done — i.e. it will execute immediately. All other
5162  * IMMED tasks are reenqueued. This means if a non-IMMED task sits at the head,
5163  * every IMMED task behind it gets reenqueued.
5164  *
5165  * Reenqueued tasks go through ops.enqueue() with %SCX_ENQ_REENQ |
5166  * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local
5167  * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers
5168  * another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in
5169  * scx_do_enqueue_task(), which ejects the task's owning scheduler. A bypassing
5170  * scheduler's tasks are placed by the kernel instead and are not counted.
5171  */
5172 static bool local_task_should_reenq(struct rq *rq, struct task_struct *p,
5173 				    u64 *reenq_flags, u32 *reason)
5174 {
5175 	bool first;
5176 
5177 	first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST);
5178 	*reenq_flags |= SCX_REENQ_TSR_NOT_FIRST;
5179 
5180 	if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || scx_task_has_kernel_slice(p)))
5181 		return false;
5182 
5183 	*reason = SCX_TASK_REENQ_KFUNC;
5184 
5185 	if ((p->scx.flags & SCX_TASK_IMMED) &&
5186 	    (!first || !(*reenq_flags & SCX_REENQ_TSR_RQ_OPEN))) {
5187 		__scx_add_event(scx_task_sched(p), SCX_EV_REENQ_IMMED, 1);
5188 		*reason = SCX_TASK_REENQ_IMMED;
5189 		return true;
5190 	}
5191 
5192 	if ((*reenq_flags & SCX_REENQ_CAP_REVOKE) &&
5193 	    scx_task_reenq_on_cap_revoke(rq, p)) {
5194 		*reason = SCX_TASK_REENQ_CAP;
5195 		return true;
5196 	}
5197 
5198 	return *reenq_flags & SCX_REENQ_ANY;
5199 }
5200 
5201 /*
5202  * The dispatcher runs ops.dequeue() and stores the final ops_state after
5203  * dropping the DSQ lock, so @p can be found on a DSQ while still
5204  * %SCX_OPSS_DISPATCHING. Wait for the dispatch to finish before operating on
5205  * @p.
5206  */
5207 void scx_wait_dispatching(struct task_struct *p)
5208 {
5209 	if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING))
5210 		wait_ops_state(p, SCX_OPSS_DISPATCHING);
5211 }
5212 
5213 static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
5214 {
5215 	LIST_HEAD(tasks);
5216 	u32 nr_enqueued = 0;
5217 	struct task_struct *p, *n;
5218 
5219 	lockdep_assert_rq_held(rq);
5220 
5221 	if (WARN_ON_ONCE(reenq_flags & __SCX_REENQ_TSR_MASK))
5222 		reenq_flags &= ~__SCX_REENQ_TSR_MASK;
5223 	if (rq_is_open(rq, 0))
5224 		reenq_flags |= SCX_REENQ_TSR_RQ_OPEN;
5225 
5226 	/*
5227 	 * The BPF scheduler may choose to dispatch tasks back to
5228 	 * @rq->scx.local_dsq. Move all candidate tasks off to a private list
5229 	 * first to avoid processing the same tasks repeatedly.
5230 	 */
5231 	list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
5232 				 scx.dsq_list.node) {
5233 		struct scx_sched *task_sch = scx_task_sched(p);
5234 		u32 reason;
5235 
5236 		/*
5237 		 * If @p is being migrated, @p's current CPU may not agree with
5238 		 * its allowed CPUs and the migration_cpu_stop is about to
5239 		 * deactivate and re-activate @p anyway. Skip re-enqueueing.
5240 		 *
5241 		 * While racing sched property changes may also dequeue and
5242 		 * re-enqueue a migrating task while its current CPU and allowed
5243 		 * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
5244 		 * the current local DSQ for running tasks and thus are not
5245 		 * visible to the BPF scheduler.
5246 		 */
5247 		if (p->migration_pending)
5248 			continue;
5249 
5250 		if (!scx_is_descendant(task_sch, sch))
5251 			continue;
5252 
5253 		if (!local_task_should_reenq(rq, p, &reenq_flags, &reason))
5254 			continue;
5255 
5256 		scx_wait_dispatching(p);
5257 		scx_dispatch_dequeue(rq, p);
5258 
5259 		WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
5260 		p->scx.flags |= reason;
5261 
5262 		list_add_tail(&p->scx.dsq_list.node, &tasks);
5263 	}
5264 
5265 	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
5266 		list_del_init(&p->scx.dsq_list.node);
5267 
5268 		scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
5269 
5270 		nr_enqueued++;
5271 	}
5272 
5273 	/*
5274 	 * The revoke that scheduled this scan may have raced the pick: donor
5275 	 * may be a now-capless task, either one that kept running or one
5276 	 * promoted off the local DSQ between the ecaps sync and this scan.
5277 	 * Zero the slice to evict it. The enqueue gate blocks new capless
5278 	 * inserts, so no later pick can slip through after the scan.
5279 	 */
5280 	if ((reenq_flags & SCX_REENQ_CAP_REVOKE) &&
5281 	    rq->donor->sched_class == &ext_sched_class &&
5282 	    scx_task_reenq_on_cap_revoke(rq, rq->donor)) {
5283 		expire_task_slice(rq->donor);
5284 		resched_curr(rq);
5285 	}
5286 
5287 	return nr_enqueued;
5288 }
5289 
5290 static void process_deferred_reenq_locals(struct rq *rq)
5291 {
5292 	lockdep_assert_rq_held(rq);
5293 
5294 	/*
5295 	 * A task can be re-queued within this loop when a reenqueued task
5296 	 * bounces straight back to the local DSQ. That recursion is bounded by
5297 	 * the per-task reenqueue cap in scx_do_enqueue_task().
5298 	 */
5299 	while (true) {
5300 		struct scx_sched *sch;
5301 		u64 reenq_flags;
5302 
5303 		scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
5304 			struct scx_deferred_reenq_local *drl =
5305 				list_first_entry_or_null(&rq->scx.deferred_reenq_locals,
5306 							 struct scx_deferred_reenq_local,
5307 							 node);
5308 			struct scx_sched_pcpu *sch_pcpu;
5309 
5310 			if (!drl)
5311 				return;
5312 
5313 			sch_pcpu = container_of(drl, struct scx_sched_pcpu,
5314 						deferred_reenq_local);
5315 			sch = sch_pcpu->sch;
5316 
5317 			reenq_flags = drl->flags;
5318 			WRITE_ONCE(drl->flags, 0);
5319 			list_del_init(&drl->node);
5320 		}
5321 
5322 		/* see schedule_dsq_reenq() */
5323 		smp_mb();
5324 
5325 		reenq_local(sch, rq, reenq_flags);
5326 	}
5327 }
5328 
5329 static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason)
5330 {
5331 	*reason = SCX_TASK_REENQ_KFUNC;
5332 	if (reenq_flags & SCX_REENQ_ANY)
5333 		return true;
5334 	return (reenq_flags & SCX_REENQ_MARKED) && (p->scx.dsq_flags & SCX_TASK_DSQ_REENQ);
5335 }
5336 
5337 static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags)
5338 {
5339 	struct rq *locked_rq = rq;
5340 	struct scx_sched *sch = dsq->sched;
5341 	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0);
5342 	struct task_struct *p;
5343 	s32 nr_visited = 0;
5344 
5345 	lockdep_assert_rq_held(rq);
5346 
5347 	raw_spin_lock(&dsq->lock);
5348 
5349 	while (likely(!READ_ONCE(sch->bypass_depth))) {
5350 		struct rq *task_rq;
5351 		u32 reason;
5352 
5353 		/*
5354 		 * Count skipped tasks too so that a long walk doesn't hold the
5355 		 * locks throughout. The cursor keeps the position.
5356 		 */
5357 		if (!(++nr_visited % SCX_TASK_ITER_BATCH)) {
5358 			raw_spin_unlock(&dsq->lock);
5359 			if (locked_rq) {
5360 				scx_rq_lock_drop(locked_rq);
5361 				raw_spin_rq_unlock(locked_rq);
5362 				locked_rq = NULL;
5363 			}
5364 			cpu_relax();
5365 			raw_spin_lock(&dsq->lock);
5366 		}
5367 
5368 		p = nldsq_cursor_next_task(&cursor, dsq);
5369 		if (!p)
5370 			break;
5371 
5372 		if (!user_task_should_reenq(p, reenq_flags, &reason))
5373 			continue;
5374 
5375 		task_rq = task_rq(p);
5376 
5377 		if (locked_rq != task_rq) {
5378 			if (locked_rq) {
5379 				scx_rq_lock_drop(locked_rq);
5380 				raw_spin_rq_unlock(locked_rq);
5381 			}
5382 			if (unlikely(!raw_spin_rq_trylock(task_rq))) {
5383 				raw_spin_unlock(&dsq->lock);
5384 				raw_spin_rq_lock(task_rq);
5385 				raw_spin_lock(&dsq->lock);
5386 			}
5387 			locked_rq = task_rq;
5388 
5389 			/* did we lose @p while switching locks? */
5390 			if (nldsq_cursor_lost_task(&cursor, task_rq, dsq, p))
5391 				continue;
5392 		}
5393 
5394 		/* @p is on @dsq, its rq and @dsq are locked */
5395 		scx_wait_dispatching(p);
5396 		dispatch_dequeue_locked(p, dsq);
5397 		raw_spin_unlock(&dsq->lock);
5398 
5399 		WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
5400 		p->scx.flags |= reason;
5401 
5402 		scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
5403 
5404 		raw_spin_lock(&dsq->lock);
5405 	}
5406 
5407 	list_del_init(&cursor.node);
5408 	raw_spin_unlock(&dsq->lock);
5409 
5410 	if (locked_rq != rq) {
5411 		if (locked_rq) {
5412 			scx_rq_lock_drop(locked_rq);
5413 			raw_spin_rq_unlock(locked_rq);
5414 		}
5415 		raw_spin_rq_lock(rq);
5416 	}
5417 }
5418 
5419 static void process_deferred_reenq_users(struct rq *rq)
5420 {
5421 	lockdep_assert_rq_held(rq);
5422 
5423 	while (true) {
5424 		struct scx_dispatch_q *dsq;
5425 		u64 dsq_id, reenq_flags;
5426 
5427 		scoped_guard (raw_spinlock, &rq->scx.deferred_reenq_lock) {
5428 			struct scx_deferred_reenq_user *dru =
5429 				list_first_entry_or_null(&rq->scx.deferred_reenq_users,
5430 							 struct scx_deferred_reenq_user,
5431 							 node);
5432 			struct scx_dsq_pcpu *dsq_pcpu;
5433 
5434 			if (!dru)
5435 				return;
5436 
5437 			dsq_pcpu = container_of(dru, struct scx_dsq_pcpu,
5438 						deferred_reenq_user);
5439 			dsq = dsq_pcpu->dsq;
5440 			reenq_flags = dru->flags;
5441 			WRITE_ONCE(dru->flags, 0);
5442 			list_del_init(&dru->node);
5443 		}
5444 
5445 		/* see schedule_dsq_reenq() */
5446 		smp_mb();
5447 
5448 		/* destroy_dsq() may have raced and invalidated @dsq, nothing to reenq */
5449 		dsq_id = READ_ONCE(dsq->id);
5450 		if (unlikely(dsq_id == SCX_DSQ_INVALID))
5451 			continue;
5452 
5453 		BUG_ON(dsq_id & SCX_DSQ_FLAG_BUILTIN);
5454 		reenq_user(rq, dsq, reenq_flags);
5455 	}
5456 }
5457 
5458 /*
5459  * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their
5460  * owning BPF schedulers choose placement again. Proxy-active tasks remain
5461  * parked and rearm the retry notification for a later proxy resolution.
5462  *
5463  * A task can be re-rejected repeatedly. Reenqueues are bounded per task by
5464  * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning
5465  * scheduler. A bypassing scheduler's tasks are placed by the kernel instead and
5466  * are not counted.
5467  */
5468 static void scx_reenq_reject(struct rq *rq)
5469 {
5470 	LIST_HEAD(tasks);
5471 	struct task_struct *p, *n;
5472 
5473 	lockdep_assert_rq_held(rq);
5474 
5475 	if (list_empty(&rq->scx.reject_dsq.list))
5476 		return;
5477 
5478 	/*
5479 	 * Move ready tasks to a private list so a task re-rejected by
5480 	 * scx_do_enqueue_task() below isn't revisited this round.
5481 	 */
5482 	list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
5483 		u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK;
5484 
5485 		WARN_ON_ONCE(!reason);
5486 
5487 		if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) {
5488 			/* Affinity machinery will dequeue and reactivate @p. */
5489 			if (p->migration_pending)
5490 				continue;
5491 
5492 			if (task_on_cpu(rq, p) || task_current_donor(rq, p)) {
5493 				rq->scx.flags |= SCX_RQ_PROXY_RETRY;
5494 				continue;
5495 			}
5496 		} else {
5497 			WARN_ON_ONCE(p->migration_pending);
5498 		}
5499 
5500 		scx_wait_dispatching(p);
5501 		scx_dispatch_dequeue(rq, p);
5502 
5503 		list_add_tail(&p->scx.dsq_list.node, &tasks);
5504 	}
5505 
5506 	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
5507 		list_del_init(&p->scx.dsq_list.node);
5508 
5509 		scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
5510 	}
5511 }
5512 
5513 static void run_deferred(struct rq *rq)
5514 {
5515 	process_ddsp_deferred_locals(rq);
5516 
5517 	if (!list_empty(&rq->scx.deferred_reenq_locals))
5518 		process_deferred_reenq_locals(rq);
5519 
5520 	if (!list_empty(&rq->scx.deferred_reenq_users))
5521 		process_deferred_reenq_users(rq);
5522 
5523 	scx_reenq_reject(rq);
5524 }
5525 
5526 #ifdef CONFIG_NO_HZ_FULL
5527 bool scx_can_stop_tick(struct rq *rq)
5528 {
5529 	struct task_struct *p = rq->donor;
5530 	struct scx_sched *sch = scx_task_sched(p);
5531 
5532 	/* Proxy execution is conservatively tick-driven for now. */
5533 	if (rq->scx.flags & SCX_RQ_PROXY_TICK)
5534 		return false;
5535 
5536 	if (p->sched_class != &ext_sched_class)
5537 		return true;
5538 
5539 	/*
5540 	 * @rq->donor may still reference an outgoing EXT task after it has been
5541 	 * dequeued. If no EXT tasks are accounted on @rq, ignore its stale
5542 	 * slice state. If another task is dispatched from a DSQ,
5543 	 * set_next_task_scx() will update the dependency for the incoming task.
5544 	 */
5545 	if (!rq->scx.nr_running)
5546 		return true;
5547 
5548 	if (scx_bypassing(sch, cpu_of(rq)))
5549 		return false;
5550 
5551 	/*
5552 	 * A running rescuee's charging and expiry are tick-driven, see
5553 	 * scx_rescue_charge(). Keep the tick while rescue is in progress.
5554 	 */
5555 	if (unlikely(p == scx_rescuee(rq)))
5556 		return false;
5557 
5558 	/*
5559 	 * @rq can dispatch from different DSQs, so we can't tell whether it
5560 	 * needs the tick or not by looking at nr_running. Allow stopping ticks
5561 	 * iff set_next_task_scx() determined that the selected scheduling context
5562 	 * can run tickless.
5563 	 */
5564 	return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;
5565 }
5566 #endif
5567 
5568 #ifdef CONFIG_EXT_GROUP_SCHED
5569 
5570 DEFINE_STATIC_PERCPU_RWSEM(scx_cgroup_ops_rwsem);
5571 
5572 void scx_tg_init(struct task_group *tg)
5573 {
5574 	tg->scx.weight = CGROUP_WEIGHT_DFL;
5575 	tg->scx.bw_period_us = default_bw_period_us();
5576 	tg->scx.bw_quota_us = RUNTIME_INF;
5577 	tg->scx.sched_idle = false;
5578 }
5579 
5580 /**
5581  * scx_tg_sched - Resolve a task_group's sched
5582  * @tg: task_group of interest
5583  *
5584  * Return the sched that @tg's ops.cgroup_init() succeeded on, %NULL if @tg
5585  * isn't inited. An autogroup tg has no cgroup of its own and resolves to the
5586  * root sched.
5587  *
5588  * When a child sched exits, its task_groups are moved to the parent and
5589  * re-inited on it. A failed re-init fails the parent in turn and leaves the
5590  * task_group without a sched it's inited on, resolving to %NULL. See
5591  * scx_cgroup_return_subtree().
5592  *
5593  * Safe for callers read-locking the ops rwsem. tg->scx.sched rewrites
5594  * write-lock it, and tg on/offline can't overlap such callers as a css's files
5595  * are created after online and drained before offline.
5596  */
5597 static struct scx_sched *scx_tg_sched(struct task_group *tg)
5598 {
5599 	lockdep_assert(lockdep_is_held(&cgroup_mutex) ||
5600 		       lockdep_is_held(&scx_cgroup_ops_rwsem));
5601 
5602 	if (!tg->css.cgroup)
5603 		tg = &root_task_group;
5604 	/* INITED means ops.cgroup_init() succeeded on @tg->scx.sched */
5605 	return (tg->scx.flags & SCX_TG_INITED) ? tg->scx.sched : NULL;
5606 }
5607 
5608 /**
5609  * scx_tg_knob_sched - Resolve the sched receiving a task_group's knob updates
5610  * @tg: task_group of interest
5611  *
5612  * Knobs of a cgroup belong to the parent. Deliver the set_* ops to the
5613  * parent task_group's sched, which equals @tg's own sched everywhere except
5614  * at a sub-scheduler attach point, where the sub's parent sched receives
5615  * them.
5616  *
5617  * Return %NULL if the parent task_group has no sched. That can happen when the
5618  * parent's ops.cgroup_init() fails while a sub-scheduler is being disabled.
5619  *
5620  * The callers sit in @tg's cgroup file writes holding the ops rwsem read
5621  * side. That extends scx_tg_sched()'s file-write argument to the parent's
5622  * sched read: a parent css outlives its children's files.
5623  */
5624 static struct scx_sched *scx_tg_knob_sched(struct task_group *tg)
5625 {
5626 	lockdep_assert(lockdep_is_held(&cgroup_mutex) ||
5627 		       lockdep_is_held(&scx_cgroup_ops_rwsem));
5628 
5629 	if (!tg->css.cgroup || !tg->css.parent)
5630 		return scx_tg_sched(&root_task_group);
5631 	return scx_tg_sched(css_tg(tg->css.parent));
5632 }
5633 
5634 int scx_tg_online(struct task_group *tg)
5635 {
5636 	int ret = 0;
5637 
5638 	WARN_ON_ONCE(tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED));
5639 
5640 	if (scx_cgroup_enabled) {
5641 		struct scx_sched *sch;
5642 
5643 		/*
5644 		 * The cgroup lifetime notifier populates cgrp->scx_sched before
5645 		 * css_online, but only on the default hierarchy. Sub-scheds are
5646 		 * attached to the cgroup2 hierarchy, so a cgroup1 task_group
5647 		 * always belongs to the root sched.
5648 		 */
5649 		if (cgroup_on_dfl(tg->css.cgroup))
5650 			sch = scx_cgroup_sched(tg->css.cgroup);
5651 		else
5652 			sch = scx_tg_sched(&root_task_group);
5653 
5654 		if (SCX_HAS_OP(sch, cgroup_init)) {
5655 			struct scx_cgroup_init_args args =
5656 				{ .weight = tg->scx.weight,
5657 				  .bw_period_us = tg->scx.bw_period_us,
5658 				  .bw_quota_us = tg->scx.bw_quota_us,
5659 				  .bw_burst_us = tg->scx.bw_burst_us,
5660 				  .sched_idle = tg->scx.sched_idle };
5661 
5662 			ret = SCX_CALL_OP_RET(sch, cgroup_init,
5663 					      NULL, tg->css.cgroup, &args);
5664 			if (ret)
5665 				ret = scx_ops_sanitize_err(sch, "cgroup_init", ret);
5666 		}
5667 		if (ret == 0) {
5668 			tg->scx.sched = sch;
5669 			tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED;
5670 		}
5671 	} else {
5672 		tg->scx.flags |= SCX_TG_ONLINE;
5673 	}
5674 
5675 	return ret;
5676 }
5677 
5678 void scx_tg_offline(struct task_group *tg)
5679 {
5680 	struct scx_sched *sch = tg->scx.sched;
5681 
5682 	WARN_ON_ONCE(!(tg->scx.flags & SCX_TG_ONLINE));
5683 
5684 	/* INITED implies non-NULL @sch, test before SCX_HAS_OP() derefs */
5685 	if (scx_cgroup_enabled && (tg->scx.flags & SCX_TG_INITED) &&
5686 	    SCX_HAS_OP(sch, cgroup_exit))
5687 		SCX_CALL_OP(sch, cgroup_exit, NULL, tg->css.cgroup);
5688 	tg->scx.sched = NULL;
5689 	tg->scx.flags &= ~(SCX_TG_ONLINE | SCX_TG_INITED);
5690 }
5691 
5692 /*
5693  * @p's sched for the cgroup migration paths. Stable as re-homes happen either
5694  * at CGROUP_TASK_MIGRATED of the same migration or under scx_cgroup_lock(),
5695  * both while holding cgroup_mutex.
5696  */
5697 static struct scx_sched *scx_cgroup_task_sched(struct task_struct *p)
5698 {
5699 	return rcu_dereference_protected(p->scx.sched, lockdep_is_held(&cgroup_mutex));
5700 }
5701 
5702 int scx_cgroup_can_attach(struct cgroup_taskset *tset)
5703 {
5704 	struct cgroup_subsys_state *css;
5705 	struct task_struct *p;
5706 	int ret;
5707 
5708 	if (!scx_cgroup_enabled)
5709 		return 0;
5710 
5711 	cgroup_taskset_for_each(p, css, tset) {
5712 		struct scx_sched *sch = scx_cgroup_task_sched(p);
5713 		struct cgroup *from = tg_cgrp(task_group(p));
5714 		struct cgroup *to = tg_cgrp(css_tg(css));
5715 
5716 		WARN_ON_ONCE(p->scx.cgrp_moving_from);
5717 
5718 		/*
5719 		 * sched_move_task() omits identity migrations. Let's match the
5720 		 * behavior so that ops.cgroup_prep_move() and ops.cgroup_move()
5721 		 * always match one-to-one.
5722 		 */
5723 		if (from == to)
5724 			continue;
5725 
5726 		/*
5727 		 * The cgroup_move ops are delivered to @p's sched, and only for
5728 		 * moves that don't re-home @p. A re-homing move changes the dfl
5729 		 * cgroup's sched and is reported through the
5730 		 * exit_task/init_task pair that the re-homing generates.
5731 		 */
5732 		if (!sch || sch != scx_cgroup_sched(task_css_set(p)->mg_dst_cset->dfl_cgrp))
5733 			continue;
5734 
5735 		if (SCX_HAS_OP(sch, cgroup_prep_move)) {
5736 			ret = SCX_CALL_OP_RET(sch, cgroup_prep_move, NULL,
5737 					      p, from, css->cgroup);
5738 			if (ret) {
5739 				ret = scx_ops_sanitize_err(sch, "cgroup_prep_move", ret);
5740 				goto err;
5741 			}
5742 		}
5743 
5744 		p->scx.cgrp_moving_from = from;
5745 	}
5746 
5747 	return 0;
5748 
5749 err:
5750 	cgroup_taskset_for_each(p, css, tset) {
5751 		struct scx_sched *sch = scx_cgroup_task_sched(p);
5752 
5753 		/* cgrp_moving_from implies non-NULL @sch, test it first */
5754 		if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move))
5755 			SCX_CALL_OP(sch, cgroup_cancel_move, NULL,
5756 				    p, p->scx.cgrp_moving_from, css->cgroup);
5757 		p->scx.cgrp_moving_from = NULL;
5758 	}
5759 
5760 	return ret;
5761 }
5762 
5763 void scx_cgroup_move_task(struct task_struct *p)
5764 {
5765 	struct scx_sched *sch;
5766 
5767 	if (!scx_cgroup_enabled)
5768 		return;
5769 
5770 	/*
5771 	 * Migration keys off css rather than cgroup identity, so it can hand an
5772 	 * unchanged-cgroup task here with cgrp_moving_from NULL. Nothing to
5773 	 * report to the BPF scheduler then, so skip it and keep prep_move and
5774 	 * move paired.
5775 	 */
5776 	sch = scx_cgroup_task_sched(p);
5777 	if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_move))
5778 		SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p),
5779 				 p, p->scx.cgrp_moving_from,
5780 				 tg_cgrp(task_group(p)));
5781 	p->scx.cgrp_moving_from = NULL;
5782 }
5783 
5784 void scx_cgroup_cancel_attach(struct cgroup_taskset *tset)
5785 {
5786 	struct cgroup_subsys_state *css;
5787 	struct task_struct *p;
5788 
5789 	if (!scx_cgroup_enabled)
5790 		return;
5791 
5792 	cgroup_taskset_for_each(p, css, tset) {
5793 		struct scx_sched *sch = scx_cgroup_task_sched(p);
5794 
5795 		/* cgrp_moving_from implies non-NULL @sch, test it first */
5796 		if (p->scx.cgrp_moving_from && SCX_HAS_OP(sch, cgroup_cancel_move))
5797 			SCX_CALL_OP(sch, cgroup_cancel_move, NULL,
5798 				    p, p->scx.cgrp_moving_from, css->cgroup);
5799 		p->scx.cgrp_moving_from = NULL;
5800 	}
5801 }
5802 
5803 void scx_group_set_weight(struct task_group *tg, unsigned long weight)
5804 {
5805 	struct scx_sched *sch;
5806 
5807 	percpu_down_read(&scx_cgroup_ops_rwsem);
5808 	sch = scx_tg_knob_sched(tg);
5809 
5810 	if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_weight) &&
5811 	    tg->scx.weight != weight)
5812 		SCX_CALL_OP(sch, cgroup_set_weight, NULL, tg_cgrp(tg), weight);
5813 
5814 	tg->scx.weight = weight;
5815 
5816 	percpu_up_read(&scx_cgroup_ops_rwsem);
5817 }
5818 
5819 void scx_group_set_idle(struct task_group *tg, bool idle)
5820 {
5821 	struct scx_sched *sch;
5822 
5823 	percpu_down_read(&scx_cgroup_ops_rwsem);
5824 	sch = scx_tg_knob_sched(tg);
5825 
5826 	if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) &&
5827 	    tg->scx.sched_idle != idle)
5828 		SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
5829 
5830 	/* Update the task group's idle state */
5831 	tg->scx.sched_idle = idle;
5832 
5833 	percpu_up_read(&scx_cgroup_ops_rwsem);
5834 }
5835 
5836 void scx_group_set_bandwidth(struct task_group *tg,
5837 			     u64 period_us, u64 quota_us, u64 burst_us)
5838 {
5839 	struct scx_sched *sch;
5840 
5841 	percpu_down_read(&scx_cgroup_ops_rwsem);
5842 	sch = scx_tg_knob_sched(tg);
5843 
5844 	if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_bandwidth) &&
5845 	    (tg->scx.bw_period_us != period_us ||
5846 	     tg->scx.bw_quota_us != quota_us ||
5847 	     tg->scx.bw_burst_us != burst_us))
5848 		SCX_CALL_OP(sch, cgroup_set_bandwidth, NULL,
5849 			    tg_cgrp(tg), period_us, quota_us, burst_us);
5850 
5851 	tg->scx.bw_period_us = period_us;
5852 	tg->scx.bw_quota_us = quota_us;
5853 	tg->scx.bw_burst_us = burst_us;
5854 
5855 	percpu_up_read(&scx_cgroup_ops_rwsem);
5856 }
5857 #endif	/* CONFIG_EXT_GROUP_SCHED */
5858 
5859 #if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED)
5860 static struct cgroup *root_cgroup(void)
5861 {
5862 	return &cgrp_dfl_root.cgrp;
5863 }
5864 
5865 /*
5866  * cgroup_lock() must nest outside the rwsem write side: a writer waiting
5867  * for cgroup_mutex deadlocks with cgroup teardown, which holds it while
5868  * draining a set_* file write blocked on the rwsem behind the writer.
5869  */
5870 void scx_cgroup_lock(void)
5871 {
5872 	cgroup_lock();
5873 #ifdef CONFIG_EXT_GROUP_SCHED
5874 	percpu_down_write(&scx_cgroup_ops_rwsem);
5875 #endif
5876 }
5877 
5878 void scx_cgroup_unlock(void)
5879 {
5880 #ifdef CONFIG_EXT_GROUP_SCHED
5881 	percpu_up_write(&scx_cgroup_ops_rwsem);
5882 #endif
5883 	cgroup_unlock();
5884 }
5885 #else	/* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
5886 static inline struct cgroup *root_cgroup(void) { return NULL; }
5887 static inline void scx_cgroup_lock(void) {}
5888 static inline void scx_cgroup_unlock(void) {}
5889 #endif	/* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
5890 
5891 /*
5892  * Omitted operations:
5893  *
5894  * - migrate_task_rq: Unnecessary as task to cpu mapping is transient.
5895  *
5896  * - task_fork/dead: We need fork/dead notifications for all tasks regardless of
5897  *   their current sched_class. Call them directly from sched core instead.
5898  */
5899 DEFINE_SCHED_CLASS(ext) = {
5900 	.enqueue_task		= enqueue_task_scx,
5901 	.dequeue_task		= dequeue_task_scx,
5902 	.yield_task		= yield_task_scx,
5903 	.yield_to_task		= yield_to_task_scx,
5904 
5905 	.wakeup_preempt		= wakeup_preempt_scx,
5906 
5907 	.pick_task		= pick_task_scx,
5908 
5909 	.put_prev_task		= put_prev_task_scx,
5910 	.set_next_task		= set_next_task_scx,
5911 
5912 	.select_task_rq		= select_task_rq_scx,
5913 	.task_woken		= task_woken_scx,
5914 	.set_cpus_allowed	= set_cpus_allowed_scx,
5915 
5916 	.task_tick		= task_tick_scx,
5917 
5918 	.switching_to		= switching_to_scx,
5919 	.switched_from		= switched_from_scx,
5920 	.switched_to		= switched_to_scx,
5921 	.reweight_task		= reweight_task_scx,
5922 	.prio_changed		= prio_changed_scx,
5923 
5924 	.update_curr		= update_curr_scx,
5925 
5926 #ifdef CONFIG_UCLAMP_TASK
5927 	.uclamp_enabled		= 1,
5928 #endif
5929 };
5930 
5931 s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch)
5932 {
5933 	s32 cpu;
5934 
5935 	memset(dsq, 0, sizeof(*dsq));
5936 
5937 	raw_spin_lock_init(&dsq->lock);
5938 	INIT_LIST_HEAD(&dsq->list);
5939 	dsq->id = dsq_id;
5940 	dsq->sched = sch;
5941 
5942 	/* per-DSQ deferred reenq state is only needed for user DSQs */
5943 	if (dsq_id & SCX_DSQ_FLAG_BUILTIN)
5944 		return 0;
5945 
5946 	dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu);
5947 	if (!dsq->pcpu_user)
5948 		return -ENOMEM;
5949 
5950 	for_each_possible_cpu(cpu) {
5951 		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
5952 
5953 		pcpu->dsq = dsq;
5954 		INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node);
5955 	}
5956 
5957 	return 0;
5958 }
5959 
5960 static void exit_dsq(struct scx_dispatch_q *dsq)
5961 {
5962 	s32 cpu;
5963 
5964 	if (!dsq->pcpu_user)
5965 		return;
5966 
5967 	for_each_possible_cpu(cpu) {
5968 		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
5969 		struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user;
5970 		struct rq *rq = cpu_rq(cpu);
5971 
5972 		/*
5973 		 * free_dsq_irq_workfn() cancelled the reenqs before the grace
5974 		 * period and schedule_dsq_reenq() queues nothing after that.
5975 		 */
5976 		if (WARN_ON_ONCE(!list_empty(&dru->node))) {
5977 			guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
5978 			list_del_init(&dru->node);
5979 		}
5980 	}
5981 
5982 	free_percpu(dsq->pcpu_user);
5983 }
5984 
5985 static void free_dsq_rcufn(struct rcu_head *rcu)
5986 {
5987 	struct scx_dispatch_q *dsq = container_of(rcu, struct scx_dispatch_q, rcu);
5988 
5989 	exit_dsq(dsq);
5990 	kfree(dsq);
5991 }
5992 
5993 static void free_dsq_irq_workfn(struct irq_work *irq_work)
5994 {
5995 	struct llist_node *to_free = llist_del_all(&dsqs_to_free);
5996 	struct scx_dispatch_q *dsq, *tmp_dsq;
5997 
5998 	llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node) {
5999 		s32 cpu;
6000 
6001 		/*
6002 		 * Cancel pending reenqs. Only a run_deferred() that started
6003 		 * before the grace period can hold one, and it keeps IRQs off,
6004 		 * so the grace period waits for it. After this sweep,
6005 		 * schedule_dsq_reenq() sees the invalid id under the same lock
6006 		 * and queues nothing.
6007 		 */
6008 		for_each_possible_cpu(cpu) {
6009 			struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
6010 			struct rq *rq = cpu_rq(cpu);
6011 
6012 			guard(raw_spinlock_irqsave)(&rq->scx.deferred_reenq_lock);
6013 			list_del_init(&pcpu->deferred_reenq_user.node);
6014 		}
6015 
6016 		call_rcu(&dsq->rcu, free_dsq_rcufn);
6017 	}
6018 }
6019 
6020 static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
6021 
6022 static void destroy_dsq(struct scx_sched *sch, u64 dsq_id)
6023 {
6024 	struct scx_dispatch_q *dsq;
6025 	unsigned long flags;
6026 
6027 	rcu_read_lock();
6028 
6029 	dsq = find_user_dsq(sch, dsq_id);
6030 	if (!dsq)
6031 		goto out_unlock_rcu;
6032 
6033 	raw_spin_lock_irqsave(&dsq->lock, flags);
6034 
6035 	if (dsq->nr) {
6036 		scx_error(sch, "attempting to destroy in-use dsq 0x%016llx (nr=%u)",
6037 			  dsq->id, dsq->nr);
6038 		goto out_unlock_dsq;
6039 	}
6040 
6041 	if (rhashtable_remove_fast(&sch->dsq_hash, &dsq->hash_node,
6042 				   dsq_hash_params))
6043 		goto out_unlock_dsq;
6044 
6045 	/*
6046 	 * Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from
6047 	 * queueing more tasks. As this function can be called from anywhere,
6048 	 * freeing is bounced through an irq work to avoid nesting RCU
6049 	 * operations inside scheduler locks.
6050 	 */
6051 	dsq->id = SCX_DSQ_INVALID;
6052 	if (llist_add(&dsq->free_node, &dsqs_to_free))
6053 		irq_work_queue(&free_dsq_irq_work);
6054 
6055 out_unlock_dsq:
6056 	raw_spin_unlock_irqrestore(&dsq->lock, flags);
6057 out_unlock_rcu:
6058 	rcu_read_unlock();
6059 }
6060 
6061 #ifdef CONFIG_EXT_GROUP_SCHED
6062 static void scx_cgroup_exit(struct scx_sched *sch)
6063 {
6064 	struct cgroup_subsys_state *css;
6065 
6066 	/*
6067 	 * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk
6068 	 * cgroups and exit all the inited ones, all online cgroups are exited.
6069 	 */
6070 	css_for_each_descendant_post(css, &root_task_group.css) {
6071 		struct task_group *tg = css_tg(css);
6072 
6073 		/* also clear the sched of tgs whose ops.cgroup_init() failed */
6074 		tg->scx.sched = NULL;
6075 		if (tg->scx.flags & SCX_TG_INITED) {
6076 			tg->scx.flags &= ~SCX_TG_INITED;
6077 			if (sch->ops.cgroup_exit)
6078 				SCX_CALL_OP(sch, cgroup_exit, NULL, css->cgroup);
6079 		}
6080 	}
6081 }
6082 
6083 static int scx_cgroup_init(struct scx_sched *sch)
6084 {
6085 	struct cgroup_subsys_state *css;
6086 	int ret;
6087 
6088 	/*
6089 	 * scx_tg_on/offline() are excluded through cgroup_lock(). If we walk
6090 	 * cgroups and init, all online cgroups are initialized.
6091 	 */
6092 	css_for_each_descendant_pre(css, &root_task_group.css) {
6093 		struct task_group *tg = css_tg(css);
6094 
6095 		if ((tg->scx.flags & (SCX_TG_ONLINE | SCX_TG_INITED)) != SCX_TG_ONLINE)
6096 			continue;
6097 
6098 		if (sch->ops.cgroup_init) {
6099 			struct scx_cgroup_init_args args = {
6100 				.weight = tg->scx.weight,
6101 				.bw_period_us = tg->scx.bw_period_us,
6102 				.bw_quota_us = tg->scx.bw_quota_us,
6103 				.bw_burst_us = tg->scx.bw_burst_us,
6104 				.sched_idle = tg->scx.sched_idle,
6105 			};
6106 
6107 			ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);
6108 			if (ret) {
6109 				scx_error(sch, "ops.cgroup_init() failed (%d)", ret);
6110 				return ret;
6111 			}
6112 		}
6113 
6114 		tg->scx.sched = sch;
6115 		tg->scx.flags |= SCX_TG_INITED;
6116 	}
6117 
6118 	return 0;
6119 }
6120 
6121 #else
6122 static void scx_cgroup_exit(struct scx_sched *sch) {}
6123 static int scx_cgroup_init(struct scx_sched *sch) { return 0; }
6124 #endif
6125 
6126 
6127 /********************************************************************************
6128  * Sysfs interface and ops enable/disable.
6129  */
6130 
6131 #define SCX_ATTR(_name)								\
6132 	static struct kobj_attribute scx_attr_##_name = {			\
6133 		.attr = { .name = __stringify(_name), .mode = 0444 },		\
6134 		.show = scx_attr_##_name##_show,				\
6135 	}
6136 
6137 static ssize_t scx_attr_state_show(struct kobject *kobj,
6138 				   struct kobj_attribute *ka, char *buf)
6139 {
6140 	return sysfs_emit(buf, "%s\n", scx_enable_state_str[scx_enable_state()]);
6141 }
6142 SCX_ATTR(state);
6143 
6144 static ssize_t scx_attr_switch_all_show(struct kobject *kobj,
6145 					struct kobj_attribute *ka, char *buf)
6146 {
6147 	return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all));
6148 }
6149 SCX_ATTR(switch_all);
6150 
6151 static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj,
6152 					 struct kobj_attribute *ka, char *buf)
6153 {
6154 	return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected));
6155 }
6156 SCX_ATTR(nr_rejected);
6157 
6158 static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj,
6159 					 struct kobj_attribute *ka, char *buf)
6160 {
6161 	return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq));
6162 }
6163 SCX_ATTR(hotplug_seq);
6164 
6165 static ssize_t scx_attr_enable_seq_show(struct kobject *kobj,
6166 					struct kobj_attribute *ka, char *buf)
6167 {
6168 	return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_enable_seq));
6169 }
6170 SCX_ATTR(enable_seq);
6171 
6172 static struct attribute *scx_global_attrs[] = {
6173 	&scx_attr_state.attr,
6174 	&scx_attr_switch_all.attr,
6175 	&scx_attr_nr_rejected.attr,
6176 	&scx_attr_hotplug_seq.attr,
6177 	&scx_attr_enable_seq.attr,
6178 	NULL,
6179 };
6180 
6181 static const struct attribute_group scx_global_attr_group = {
6182 	.attrs = scx_global_attrs,
6183 };
6184 
6185 static void free_pnode(struct scx_sched_pnode *pnode);
6186 static void free_exit_info(struct scx_exit_info *ei);
6187 static const char *scx_exit_reason(enum scx_exit_kind kind);
6188 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
6189 
6190 s32 scx_alloc_kern_arena_objs(struct scx_sched *sch)
6191 {
6192 	size_t size = struct_size_t(struct scx_cmask, bits,
6193 				    SCX_CMASK_NR_WORDS(num_possible_cpus()));
6194 	struct scx_cmask *online;
6195 	struct scx_cmask_ref ref;
6196 	int cpu;
6197 
6198 	/* hotplug stays excluded until the online mask is published */
6199 	lockdep_assert_cpus_held();
6200 
6201 	if (!sch->is_cid_type || !sch->arena_pool)
6202 		return 0;
6203 
6204 	sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *);
6205 	if (!sch->set_cmask_scratch)
6206 		return -ENOMEM;
6207 
6208 	for_each_possible_cpu(cpu) {
6209 		struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
6210 
6211 		*slot = scx_arena_alloc(sch, size);
6212 		if (!*slot)
6213 			return -ENOMEM;
6214 		scx_cmask_init(*slot, 0, num_possible_cpus());
6215 	}
6216 
6217 	/* pack the online mask alongside the scratch masks */
6218 	online = scx_arena_alloc(sch, size);
6219 	if (!online)
6220 		return -ENOMEM;
6221 
6222 	scoped_guard(rcu) {
6223 		scx_cmask_ref_init_kern(sch, online, 0, num_possible_cpus(), &ref);
6224 		scx_cmask_ref_from_cpumask(&ref, cpu_active_mask);
6225 	}
6226 	sch->online_cmask = online;
6227 
6228 	return 0;
6229 }
6230 
6231 static void scx_free_kern_arena_objs(struct scx_sched *sch)
6232 {
6233 	size_t size = struct_size_t(struct scx_cmask, bits,
6234 				    SCX_CMASK_NR_WORDS(num_possible_cpus()));
6235 	int cpu;
6236 
6237 	scx_arena_free(sch, sch->online_cmask, size);
6238 	if (!sch->set_cmask_scratch)
6239 		return;
6240 
6241 	for_each_possible_cpu(cpu) {
6242 		struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
6243 
6244 		scx_arena_free(sch, *slot, size);
6245 	}
6246 	free_percpu(sch->set_cmask_scratch);
6247 	sch->set_cmask_scratch = NULL;
6248 }
6249 
6250 static void scx_sched_free_rcu_work(struct work_struct *work)
6251 {
6252 	struct rcu_work *rcu_work = to_rcu_work(work);
6253 	struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work);
6254 	struct rhashtable_iter rht_iter;
6255 	struct scx_dispatch_q *dsq;
6256 	int cpu, node;
6257 
6258 	free_cpumask_var(sch->bypass_lb_donee_cpumask);
6259 	free_cpumask_var(sch->bypass_lb_resched_cpumask);
6260 	free_cpumask_var(sch->stall_cpus);
6261 
6262 #ifdef CONFIG_EXT_SUB_SCHED
6263 	kfree(sch->cgrp_path);
6264 	if (sch_cgroup(sch))
6265 		cgroup_put(sch_cgroup(sch));
6266 	if (sch->sub_kset)
6267 		kobject_put(&sch->sub_kset->kobj);
6268 	if (scx_parent(sch))
6269 		kobject_put(&scx_parent(sch)->kobj);
6270 #endif	/* CONFIG_EXT_SUB_SCHED */
6271 
6272 	for_each_possible_cpu(cpu) {
6273 		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
6274 
6275 		free_cpumask_var(pcpu->cpus_to_kick);
6276 		free_cpumask_var(pcpu->cpus_to_kick_if_idle);
6277 		free_cpumask_var(pcpu->cpus_to_preempt);
6278 		free_cpumask_var(pcpu->cpus_to_preempt_lazy);
6279 		free_cpumask_var(pcpu->cpus_to_wait);
6280 
6281 		exit_dsq(scx_bypass_dsq(sch, cpu));
6282 	}
6283 
6284 	free_percpu(sch->pcpu);
6285 
6286 	for_each_node_state(node, N_POSSIBLE)
6287 		free_pnode(sch->pnode[node]);
6288 	kfree(sch->pnode);
6289 
6290 	scx_free_pshards(sch);
6291 
6292 	rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);
6293 	do {
6294 		rhashtable_walk_start(&rht_iter);
6295 
6296 		while (!IS_ERR_OR_NULL((dsq = rhashtable_walk_next(&rht_iter))))
6297 			destroy_dsq(sch, dsq->id);
6298 
6299 		rhashtable_walk_stop(&rht_iter);
6300 	} while (dsq == ERR_PTR(-EAGAIN));
6301 	rhashtable_walk_exit(&rht_iter);
6302 
6303 	rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
6304 	free_exit_info(sch->exit_info);
6305 	scx_free_kern_arena_objs(sch);
6306 	scx_arena_pool_destroy(sch);
6307 	if (sch->arena_map)
6308 		bpf_map_put(sch->arena_map);
6309 
6310 	/* @sch is completely inactive by now */
6311 	scx_dec_has_subs(sch);
6312 
6313 	kfree(sch);
6314 }
6315 
6316 /**
6317  * scx_sched_quiesce_rcu_work - Quiesce a sched and queue its free
6318  * @work: &scx_sched.rcu_work
6319  *
6320  * $sch is freed after two RCU grace periods because of its ecaps sync nodes.
6321  *
6322  * The first grace period runs between scx_kobj_release() and this function. It
6323  * waits for the paths that found $sch before scx_unlink_sched(), such as the
6324  * sub-cap kfuncs and the cpu hotplug hooks. Until they finish, they can queue
6325  * $sch's syncs, and a cpu can take a queued node into a sync batch. The first
6326  * grace period doesn't wait for a batch that started after it began.
6327  *
6328  * Nothing queues $sch's syncs now. Remove the nodes still on the cpus' lists. A
6329  * batch that took a node can still write $sch's pcpu, and a delivery in the
6330  * batch can drop the rq lock before that write. The batch keeps irqs off from
6331  * taking the node to its last pcpu write, so the second grace period waits for
6332  * it before scx_sched_free_rcu_work() frees $sch.
6333  */
6334 static void scx_sched_quiesce_rcu_work(struct work_struct *work)
6335 {
6336 	struct rcu_work *rcu_work = to_rcu_work(work);
6337 	struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work);
6338 	s32 cpu;
6339 
6340 	irq_work_sync(&sch->propagate_exit_irq_work);
6341 	irq_work_sync(&sch->disable_irq_work);
6342 	kthread_destroy_worker(sch->helper);
6343 	timer_shutdown_sync(&sch->bypass_lb_timer);
6344 
6345 	for_each_possible_cpu(cpu) {
6346 		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
6347 
6348 		/*
6349 		 * $sch would have entered bypass mode before the first RCU
6350 		 * grace period. As that blocks new deferrals, all
6351 		 * deferred_reenq_local_node's must be off-list by now.
6352 		 */
6353 		WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
6354 
6355 		/* remove the queued ecaps sync so the pcpu can be freed */
6356 		scx_discard_ecaps_to_sync(cpu, pcpu);
6357 
6358 		/*
6359 		 * Bypass blocks new kicks. Flush the kick irq_work so this
6360 		 * pcpu's to_kick_node is off the list before it is freed.
6361 		 */
6362 		irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
6363 		WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node));
6364 	}
6365 
6366 	INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work);
6367 	queue_rcu_work(system_dfl_wq, &sch->rcu_work);
6368 }
6369 
6370 static void scx_kobj_release(struct kobject *kobj)
6371 {
6372 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
6373 
6374 	INIT_RCU_WORK(&sch->rcu_work, scx_sched_quiesce_rcu_work);
6375 	queue_rcu_work(system_dfl_wq, &sch->rcu_work);
6376 }
6377 
6378 static ssize_t scx_attr_ops_show(struct kobject *kobj,
6379 				 struct kobj_attribute *ka, char *buf)
6380 {
6381 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
6382 
6383 	return sysfs_emit(buf, "%s\n", sch->ops.name);
6384 }
6385 SCX_ATTR(ops);
6386 
6387 #define scx_attr_event_show(buf, at, events, kind) ({				\
6388 	sysfs_emit_at(buf, at, "%s %llu\n", #kind, (events)->kind);		\
6389 })
6390 
6391 static ssize_t scx_attr_events_show(struct kobject *kobj,
6392 				    struct kobj_attribute *ka, char *buf)
6393 {
6394 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
6395 	struct scx_event_stats events;
6396 	int at = 0;
6397 
6398 	scx_read_events(sch, &events);
6399 #define SCX_EVENT(name)	(at += scx_attr_event_show(buf, at, &events, name))
6400 	SCX_EVENTS_LIST(SCX_EVENT);
6401 #undef SCX_EVENT
6402 	return at;
6403 }
6404 SCX_ATTR(events);
6405 
6406 #ifdef CONFIG_EXT_SUB_SCHED
6407 static const char *scx_cap_names[__SCX_NR_CAPS] = {
6408 	[__SCX_CAP_ENQ_IMMED]	= "enq_immed",
6409 	[__SCX_CAP_ENQ]		= "enq",
6410 	[__SCX_CAP_PREEMPT]	= "preempt",
6411 	[__SCX_CAP_PERF]	= "perf",
6412 };
6413 
6414 static ssize_t scx_attr_caps_show(struct kobject *kobj,
6415 				  struct kobj_attribute *ka, char *buf)
6416 {
6417 	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
6418 	u32 npossible = num_possible_cpus();
6419 	struct scx_cmask *agg __free(kfree) =
6420 		kzalloc_flex(*agg, bits, SCX_CMASK_NR_WORDS(npossible));
6421 	unsigned long *agg_bm __free(bitmap) = bitmap_zalloc(npossible, GFP_KERNEL);
6422 	ssize_t count = 0;
6423 	s32 cap, si;
6424 
6425 	if (!agg || !agg_bm)
6426 		return -ENOMEM;
6427 
6428 	for (cap = 0; cap < __SCX_NR_CAPS; cap++) {
6429 		SCX_CMASK_DEFINE(snap, 0, SCX_CID_SHARD_MAX_CPUS);
6430 
6431 		scx_cmask_init(agg, 0, npossible);
6432 		for (si = 0; si < sch->nr_pshards; si++) {
6433 			struct scx_cmask *cm = &sch->pshard[si]->caps[cap].cmask;
6434 
6435 			scx_cmask_reframe(snap, cm->base, cm->nr_cids);
6436 			scx_cmask_copy(snap, cm);
6437 			scx_cmask_or(agg, snap);
6438 		}
6439 		/* %*pbl takes unsigned long bitmap layout, convert from u64 */
6440 		bitmap_from_arr64(agg_bm, agg->bits, npossible);
6441 		count += sysfs_emit_at(buf, count, "%s: %*pbl\n",
6442 				       scx_cap_names[cap], npossible, agg_bm);
6443 	}
6444 	return count;
6445 }
6446 SCX_ATTR(caps);
6447 #endif	/* CONFIG_EXT_SUB_SCHED */
6448 
6449 static struct attribute *scx_sched_attrs[] = {
6450 	&scx_attr_ops.attr,
6451 	&scx_attr_events.attr,
6452 #ifdef CONFIG_EXT_SUB_SCHED
6453 	&scx_attr_caps.attr,
6454 #endif
6455 	NULL,
6456 };
6457 ATTRIBUTE_GROUPS(scx_sched);
6458 
6459 static const struct kobj_type scx_ktype = {
6460 	.release = scx_kobj_release,
6461 	.sysfs_ops = &kobj_sysfs_ops,
6462 	.default_groups = scx_sched_groups,
6463 };
6464 
6465 static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)
6466 {
6467 	const struct scx_sched *sch;
6468 
6469 	/*
6470 	 * scx_uevent() can be reached by both scx_sched kobjects (scx_ktype)
6471 	 * and sub-scheduler kset kobjects (kset_ktype) through the parent
6472 	 * chain walk. Filter out the latter to avoid invalid casts.
6473 	 */
6474 	if (kobj->ktype != &scx_ktype)
6475 		return 0;
6476 
6477 	sch = container_of(kobj, struct scx_sched, kobj);
6478 
6479 	return add_uevent_var(env, "SCXOPS=%s", sch->ops.name);
6480 }
6481 
6482 static const struct kset_uevent_ops scx_uevent_ops = {
6483 	.uevent = scx_uevent,
6484 };
6485 
6486 /*
6487  * Used by sched_fork() and __setscheduler_class() to pick the matching
6488  * sched_class. dl/rt are already handled.
6489  */
6490 bool task_should_scx(int policy)
6491 {
6492 	/* if disabled, nothing should be on it */
6493 	if (!scx_enabled())
6494 		return false;
6495 
6496 	/* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */
6497 	if (READ_ONCE(scx_switching_all))
6498 		return true;
6499 
6500 	/*
6501 	 * scx is tearing down - keep new SCHED_EXT tasks out.
6502 	 *
6503 	 * Must come after scx_switching_all test, which serves as a proxy
6504 	 * for __scx_switched_all. While __scx_switched_all is set, we must
6505 	 * return true via the branch above: a fork routed to fair would
6506 	 * stall because next_active_class() skips fair.
6507 	 *
6508 	 * This can develop into a deadlock - scx holds scx_enable_mutex across
6509 	 * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is
6510 	 * the stalled task, the disable path can never grab the mutex to clear
6511 	 * scx_switching_all.
6512 	 */
6513 	if (unlikely(scx_enable_state() == SCX_DISABLING))
6514 		return false;
6515 
6516 	return policy == SCHED_EXT;
6517 }
6518 
6519 bool scx_allow_ttwu_queue(const struct task_struct *p)
6520 {
6521 	struct scx_sched *sch;
6522 
6523 	if (!scx_enabled())
6524 		return true;
6525 
6526 	sch = scx_task_sched(p);
6527 	if (unlikely(!sch))
6528 		return true;
6529 
6530 	if (sch->ops.flags & SCX_OPS_ALLOW_QUEUED_WAKEUP)
6531 		return true;
6532 
6533 	if (unlikely(p->sched_class != &ext_sched_class))
6534 		return true;
6535 
6536 	return false;
6537 }
6538 
6539 /**
6540  * handle_lockup - sched_ext common lockup handler
6541  * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current.
6542  * @fmt: format string
6543  *
6544  * Called on system stall or lockup condition and initiates abort of sched_ext
6545  * if enabled, which may resolve the reported lockup.
6546  *
6547  * Returns %true if sched_ext is enabled and abort was initiated, which may
6548  * resolve the lockup. %false if sched_ext is not enabled or abort was already
6549  * initiated by someone else.
6550  */
6551 static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...)
6552 {
6553 	struct scx_sched *sch;
6554 	va_list args;
6555 	bool ret;
6556 
6557 	guard(rcu)();
6558 
6559 	sch = rcu_dereference(scx_root);
6560 	if (unlikely(!sch))
6561 		return false;
6562 
6563 	switch (scx_enable_state()) {
6564 	case SCX_ENABLING:
6565 	case SCX_ENABLED:
6566 		va_start(args, fmt);
6567 		ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args);
6568 		va_end(args);
6569 		return ret;
6570 	default:
6571 		return false;
6572 	}
6573 }
6574 
6575 /**
6576  * scx_rcu_cpu_stall - sched_ext RCU CPU stall handler
6577  * @stalled_mask: bit mask of stalled CPUs
6578  *
6579  * While there are various reasons why RCU CPU stalls can occur on a system
6580  * that may not be caused by the current BPF scheduler, try kicking out the
6581  * current scheduler in an attempt to recover the system to a good state before
6582  * issuing panics.
6583  *
6584  * Returns %true if sched_ext is enabled and abort was initiated, which may
6585  * resolve the reported RCU stall. %false if sched_ext is not enabled or someone
6586  * else already initiated abort.
6587  */
6588 bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask)
6589 {
6590 	struct scx_sched *sch;
6591 	struct scx_exit_info *ei;
6592 	int exit_cpu;
6593 
6594 	guard(rcu)();
6595 
6596 	sch = rcu_dereference(scx_root);
6597 	if (unlikely(!sch))
6598 		return false;
6599 
6600 	switch (scx_enable_state()) {
6601 	case SCX_ENABLING:
6602 	case SCX_ENABLED:
6603 		break;
6604 	default:
6605 		return false;
6606 	}
6607 
6608 	exit_cpu = cpumask_empty(stalled_mask) ? -1 : (int)cpumask_first(stalled_mask);
6609 	ei = sch->exit_info;
6610 
6611 	guard(preempt)();
6612 
6613 	if (!scx_claim_exit(sch, SCX_EXIT_ERROR))
6614 		return false;
6615 
6616 #ifdef CONFIG_STACKTRACE
6617 	ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
6618 #endif
6619 	scnprintf(ei->msg, SCX_EXIT_MSG_LEN, "RCU CPU stall on CPUs (%*pbl)",
6620 		  cpumask_pr_args(stalled_mask));
6621 	ei->kind = SCX_EXIT_ERROR;
6622 	ei->reason = scx_exit_reason(SCX_EXIT_ERROR);
6623 	ei->exit_cpu = exit_cpu;
6624 	cpumask_copy(sch->stall_cpus, stalled_mask);
6625 
6626 	irq_work_queue(&sch->disable_irq_work);
6627 	return true;
6628 }
6629 
6630 /**
6631  * scx_softlockup - sched_ext softlockup handler
6632  * @dur_s: number of seconds of CPU stuck due to soft lockup
6633  *
6634  * On some multi-socket setups (e.g. 2x Intel 8480c), the BPF scheduler can
6635  * live-lock the system by making many CPUs target the same DSQ to the point
6636  * where soft-lockup detection triggers. This function is called from
6637  * soft-lockup watchdog when the triggering point is close and tries to unjam
6638  * the system and aborting the BPF scheduler.
6639  */
6640 void scx_softlockup(u32 dur_s)
6641 {
6642 	int cpu = smp_processor_id();
6643 
6644 	if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s))
6645 		return;
6646 
6647 	printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n",
6648 			cpu, dur_s);
6649 }
6650 
6651 /**
6652  * scx_hardlockup - sched_ext hardlockup handler
6653  * @cpu: the target CPU
6654  *
6655  * A poorly behaving BPF scheduler can trigger hard lockup by e.g. putting
6656  * numerous affinitized tasks in a single queue and directing all CPUs at it.
6657  * Try kicking out the current scheduler in an attempt to recover the system to
6658  * a good state before taking more drastic actions.
6659  *
6660  * Called from NMI. Aborting the scheduler sets ->aborting throughout the
6661  * hierarchy before returning, which is what breaks the dispatch-path live-locks
6662  * that can hard-lock CPUs.
6663  *
6664  * Returns %true if sched_ext is enabled and abort was initiated, which may
6665  * resolve the lockup. %false if sched_ext is not enabled or abort was already
6666  * initiated by someone else.
6667  */
6668 bool scx_hardlockup(int cpu)
6669 {
6670 	if (!handle_lockup(cpu, "hard lockup - CPU %d", cpu))
6671 		return false;
6672 
6673 	printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
6674 			cpu);
6675 	return true;
6676 }
6677 
6678 /**
6679  * bypass_lb_reenq_home - Reenqueue a task at its home from the balancer
6680  * @rq: @p's home rq, locked
6681  * @p: task off any DSQ, taken from the bypass DSQ of @dsq_cpu
6682  * @dsq_cpu: cpu whose bypass DSQ gave up @p
6683  *
6684  * @p came off the bypass DSQ of @dsq_cpu with no donee able to take it. A drain
6685  * sends a migration-disabled @p here unless the donee picked for it is its
6686  * home, as every other cpu refuses it. The enqueue sends @p to rescue when the
6687  * host lacks %SCX_CAP_BASE at home and rescue is enabled, and to the home's
6688  * bypass DSQ otherwise. When @p must run at home, migration-disabled or with a
6689  * migration pending, and the host lacks BASE there, it is admitted to the
6690  * home's local DSQ in either case, see bypass_enq_target_dsq(). No DSQ lock may
6691  * be held, as the enqueue takes one.
6692  */
6693 static void bypass_lb_reenq_home(struct rq *rq, struct task_struct *p, s32 dsq_cpu)
6694 {
6695 	lockdep_assert_rq_held(rq);
6696 
6697 	WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
6698 	p->scx.flags |= SCX_TASK_REENQ_CAP;
6699 	p->scx.reenq_reason_caps = SCX_CAP_BASE;
6700 	p->scx.reenq_reason_cid = __scx_cpu_to_cid(dsq_cpu);
6701 	scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
6702 }
6703 
6704 /**
6705  * bypass_lb_can_move - Test whether @p can move to the bypass DSQ of @donee
6706  * @sch: host sched whose bypass DSQs are balanced
6707  * @p: task on a bypass DSQ of @sch, homed anywhere
6708  * @donee: cpu whose bypass DSQ would receive @p, -1 for none
6709  *
6710  * @p's rq lock pins @p's affinity. Without it the answer is a hint. A task
6711  * whose donee is its own cpu can always run there, and
6712  * task_can_run_on_remote_rq() warns when asked about a task's own cpu.
6713  */
6714 static bool bypass_lb_can_move(struct scx_sched *sch, struct task_struct *p, s32 donee)
6715 {
6716 	return donee >= 0 && (donee == task_cpu(p) ||
6717 			      task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false));
6718 }
6719 
6720 /**
6721  * bypass_lb_move - Move one task off a bypass DSQ for the balancer
6722  * @sch: host sched whose bypass DSQs are balanced
6723  * @donor_rq: rq of the cpu whose bypass DSQ gives up @p, locked
6724  * @donor_dsq: @sch's bypass DSQ on @donor_rq's cpu, locked
6725  * @p: task on @donor_dsq, homed anywhere
6726  * @donee: cpu whose bypass DSQ receives @p, -1 for none
6727  *
6728  * Take @p off @donor_dsq and insert it into the bypass DSQ of @donee. The
6729  * caller tests bypass_lb_can_move() without @p's rq lock for a task homed
6730  * elsewhere, and not at all for a drain, so the test is repeated here under
6731  * @p's rq lock. A task that fails it is not put back, as the host may no longer
6732  * consume @donor_dsq. bypass_lb_reenq_home() reenqueues it at its home instead.
6733  *
6734  * The insert and the reenqueue run under @p's rq lock. A task homed on
6735  * @donor_rq is dequeued in place. unlink_dsq_and_switch_rq_lock() takes one
6736  * homed elsewhere off and switches the rq lock from @donor_rq to its own. The
6737  * switch can lose to a dequeue, and @p then belongs to the dequeuer.
6738  *
6739  * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is
6740  * unlocked in between, and @donor_rq too while the lock is switched, so the
6741  * caller's walk must find its place in @donor_dsq again after the call.
6742  *
6743  * Return the cpu that received @p, or -1 if @p was lost to a dequeue.
6744  */
6745 static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq,
6746 			  struct scx_dispatch_q *donor_dsq, struct task_struct *p, s32 donee)
6747 {
6748 	struct rq *rq = task_rq(p);
6749 	s32 ret;
6750 
6751 	lockdep_assert_rq_held(donor_rq);
6752 	lockdep_assert_held(&donor_dsq->lock);
6753 
6754 	if (rq == donor_rq) {
6755 		scx_wait_dispatching(p);
6756 		dispatch_dequeue_locked(p, donor_dsq);
6757 		raw_spin_unlock(&donor_dsq->lock);
6758 	} else if (!unlink_dsq_and_switch_rq_lock(p, donor_dsq, donor_rq, rq)) {
6759 		/* lost to a dequeue, @p is the dequeuer's now */
6760 		ret = -1;
6761 		goto out;
6762 	}
6763 
6764 	if (bypass_lb_can_move(sch, p, donee)) {
6765 		scx_dispatch_enqueue(sch, rq, scx_bypass_dsq(sch, donee), p, 0, 0, 0);
6766 		ret = donee;
6767 	} else {
6768 		/*
6769 		 * No donee, or @p can't run on it. Reenqueue @p at home
6770 		 * instead, see bypass_lb_reenq_home().
6771 		 */
6772 		bypass_lb_reenq_home(rq, p, cpu_of(donor_rq));
6773 		ret = cpu_of(rq);
6774 	}
6775 out:
6776 	if (rq != donor_rq)
6777 		switch_rq_lock(rq, donor_rq);
6778 	raw_spin_lock(&donor_dsq->lock);
6779 	return ret;
6780 }
6781 
6782 static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
6783 			 struct cpumask *donee_mask, struct cpumask *resched_mask,
6784 			 u32 nr_donor_target, u32 nr_donee_target)
6785 {
6786 	struct rq *donor_rq = cpu_rq(donor);
6787 	struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor);
6788 	struct task_struct *p, *n;
6789 	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0);
6790 	s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target;
6791 	u32 nr_balanced = 0, min_delta_us;
6792 
6793 	/*
6794 	 * All we want to guarantee is reasonable forward progress. No reason to
6795 	 * fine tune. Assuming every task on @donor_dsq runs their full slice,
6796 	 * consider offloading iff the total queued duration is over the
6797 	 * threshold. A zero target is a drain and empties the DSQ regardless.
6798 	 */
6799 	min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV;
6800 	if (nr_donor_target &&
6801 	    delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us)))
6802 		return 0;
6803 
6804 	raw_spin_rq_lock_irq(donor_rq);
6805 	raw_spin_lock(&donor_dsq->lock);
6806 	/* bound the walk to the tasks queued now, see the seq test below */
6807 	cursor.priv = donor_dsq->seq;
6808 	list_add(&cursor.node, &donor_dsq->list);
6809 resume:
6810 	n = container_of(&cursor, struct task_struct, scx.dsq_list);
6811 	n = nldsq_next_task(donor_dsq, n, false);
6812 
6813 	while ((p = n)) {
6814 		s32 donee, cpu;
6815 
6816 		n = nldsq_next_task(donor_dsq, n, false);
6817 
6818 		if (donor_dsq->nr <= nr_donor_target)
6819 			break;
6820 
6821 		/*
6822 		 * Without a donee, a balancing donor has nothing to do. A drain
6823 		 * still sends its tasks home.
6824 		 */
6825 		if (cpumask_empty(donee_mask) && nr_donor_target)
6826 			break;
6827 
6828 		/*
6829 		 * A task queued after the walk began, including one the walk
6830 		 * itself reenqueued, is left for the next pass.
6831 		 */
6832 		if (u32_before(cursor.priv, p->scx.dsq_seq))
6833 			continue;
6834 
6835 		/*
6836 		 * A balancing donor leaves a task that can't move where it is.
6837 		 * A home reenqueue would push one homed on the donor to the
6838 		 * back of @donor_dsq on every pass. A drain sends it home.
6839 		 */
6840 		donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr);
6841 		if (donee >= nr_cpu_ids)
6842 			donee = -1;
6843 		if (nr_donor_target && !bypass_lb_can_move(sch, p, donee))
6844 			continue;
6845 
6846 		/*
6847 		 * The move drops @donor_dsq's lock, and the rq lock too when
6848 		 * @p is homed elsewhere. The donor cpu's dispatch and its
6849 		 * enqueuers can reach @donor_dsq then. Park the cursor behind
6850 		 * @p and resume from it afterwards. The seq test above keeps a
6851 		 * task enqueued meanwhile out of the pass.
6852 		 */
6853 		list_move(&cursor.node, &p->scx.dsq_list.node);
6854 		cpu = bypass_lb_move(sch, donor_rq, donor_dsq, p, donee);
6855 		if (cpu >= 0) {
6856 			/*
6857 			 * @cpu might have been idle and need to be woken up.
6858 			 * No need to be clever. Reschedule every cpu that
6859 			 * receives tasks.
6860 			 */
6861 			cpumask_set_cpu(cpu, resched_mask);
6862 
6863 			if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) >= nr_donee_target)
6864 				cpumask_clear_cpu(cpu, donee_mask);
6865 
6866 			/*
6867 			 * The rq lock is held across the walk, not cycled per
6868 			 * move. @donor_dsq can be very long and contended, and
6869 			 * the rq lock is what keeps its consumer and enqueuers
6870 			 * away while the walk cycles the DSQ lock. The rare
6871 			 * move of a task homed elsewhere does switch it away,
6872 			 * see the cursor comment above. The walk can still take
6873 			 * a while, so let the cpu breathe every so often.
6874 			 */
6875 			if (!(++nr_balanced % SCX_BYPASS_LB_BATCH)) {
6876 				raw_spin_unlock(&donor_dsq->lock);
6877 				scx_rq_lock_drop(donor_rq);
6878 				raw_spin_rq_unlock_irq(donor_rq);
6879 				cpu_relax();
6880 				raw_spin_rq_lock_irq(donor_rq);
6881 				raw_spin_lock(&donor_dsq->lock);
6882 			}
6883 		}
6884 		goto resume;
6885 	}
6886 
6887 	list_del_init(&cursor.node);
6888 	raw_spin_unlock(&donor_dsq->lock);
6889 	scx_rq_lock_drop(donor_rq);
6890 	raw_spin_rq_unlock_irq(donor_rq);
6891 
6892 	return nr_balanced;
6893 }
6894 
6895 static void bypass_lb_node(struct scx_sched *sch, int node)
6896 {
6897 	const struct cpumask *node_mask = cpumask_of_node(node);
6898 	struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask;
6899 	struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask;
6900 	u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us);
6901 	u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0;
6902 	u32 nr_target, nr_donor_target;
6903 	u32 before_min = U32_MAX, before_max = 0;
6904 	u32 after_min = U32_MAX, after_max = 0;
6905 	int cpu;
6906 
6907 	/* count the target tasks and CPUs */
6908 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6909 		u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr);
6910 
6911 		nr_tasks += nr;
6912 		/* only a cpu whose dispatch consumes its DSQ is capacity */
6913 		if (!scx_missing_caps(sch, cpu, SCX_CAP_BASE))
6914 			nr_cpus++;
6915 
6916 		before_min = min(nr, before_min);
6917 		before_max = max(nr, before_max);
6918 	}
6919 
6920 	if (!nr_tasks)
6921 		return;
6922 
6923 	/*
6924 	 * We don't want CPUs to have more than $nr_donor_target tasks and
6925 	 * balancing to fill donee CPUs up to $nr_target. Once targets are
6926 	 * calculated, find the donee CPUs.
6927 	 */
6928 	nr_target = nr_cpus ? DIV_ROUND_UP(nr_tasks, nr_cpus) : 0;
6929 	nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100);
6930 
6931 	cpumask_clear(donee_mask);
6932 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6933 		/* a donee's DSQ is consumed only if @sch holds %SCX_CAP_BASE */
6934 		if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target &&
6935 		    !scx_missing_caps(sch, cpu, SCX_CAP_BASE))
6936 			cpumask_set_cpu(cpu, donee_mask);
6937 	}
6938 
6939 	/*
6940 	 * Offload the !donee cpus. A cpu where the host lost %SCX_CAP_BASE no
6941 	 * longer consumes its DSQ. Drain it as a donor with a target of zero. A
6942 	 * zero interval disables the balancing, not the drain.
6943 	 */
6944 	cpumask_clear(resched_mask);
6945 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6946 		u32 target = nr_donor_target;
6947 
6948 		if (unlikely(scx_missing_caps(sch, cpu, SCX_CAP_BASE))) {
6949 			/* the mask may predate the loss, keep this cpu out */
6950 			cpumask_clear_cpu(cpu, donee_mask);
6951 			target = 0;
6952 		} else if (!intv_us || cpumask_empty(donee_mask) ||
6953 			   cpumask_test_cpu(cpu, donee_mask)) {
6954 			continue;
6955 		}
6956 		if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= target)
6957 			continue;
6958 
6959 		nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask,
6960 					     target, nr_target);
6961 	}
6962 
6963 	for_each_cpu(cpu, resched_mask)
6964 		resched_cpu(cpu);
6965 
6966 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
6967 		u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr);
6968 
6969 		after_min = min(nr, after_min);
6970 		after_max = max(nr, after_max);
6971 
6972 	}
6973 
6974 	trace_sched_ext_bypass_lb(node, nr_cpus, nr_tasks, nr_balanced,
6975 				  before_min, before_max, after_min, after_max);
6976 }
6977 
6978 /*
6979  * In bypass mode, all tasks are put on the per-CPU bypass DSQs. If the machine
6980  * is over-saturated and the BPF scheduler skewed tasks into few CPUs, some
6981  * bypass DSQs can be overloaded. If there are enough tasks to saturate other
6982  * lightly loaded CPUs, such imbalance can lead to very high execution latency
6983  * on the overloaded CPUs and thus to hung tasks and RCU stalls. To avoid such
6984  * outcomes, a simple load balancing mechanism is implemented by the following
6985  * timer which runs periodically while bypass mode is in effect.
6986  */
6987 static void scx_bypass_lb_timerfn(struct timer_list *timer)
6988 {
6989 	struct scx_sched *sch = container_of(timer, struct scx_sched, bypass_lb_timer);
6990 	int node;
6991 	u32 intv_us;
6992 
6993 	if (!scx_bypass_dsp_enabled(sch))
6994 		return;
6995 
6996 	for_each_node_with_cpus(node)
6997 		bypass_lb_node(sch, node);
6998 
6999 	/*
7000 	 * A kick during this run can leave the timer pending at now, see
7001 	 * scx_bypass_lb_kick(). timer_reduce() only brings the expiry forward,
7002 	 * so that kick is kept.
7003 	 */
7004 	intv_us = READ_ONCE(scx_bypass_lb_intv_us);
7005 	if (intv_us)
7006 		timer_reduce(timer, jiffies + usecs_to_jiffies(intv_us));
7007 }
7008 
7009 /**
7010  * scx_bypass_lb_kick - Run the bypass load balancer now
7011  * @sch: sched that lost %SCX_CAP_BASE on a cpu
7012  *
7013  * @sch no longer consumes its bypass DSQ on that cpu. Have the balancer drain
7014  * it without waiting for the next interval, see bypass_lb_node(). A loss during
7015  * a run leaves the timer pending at now, and the run that follows sees it.
7016  */
7017 void scx_bypass_lb_kick(struct scx_sched *sch)
7018 {
7019 	if (scx_bypass_dsp_enabled(sch))
7020 		mod_timer(&sch->bypass_lb_timer, jiffies);
7021 }
7022 
7023 static bool inc_bypass_depth(struct scx_sched *sch)
7024 {
7025 	lockdep_assert_held(&scx_bypass_lock);
7026 
7027 	WARN_ON_ONCE(sch->bypass_depth < 0);
7028 	WRITE_ONCE(sch->bypass_depth, sch->bypass_depth + 1);
7029 	if (sch->bypass_depth != 1)
7030 		return false;
7031 
7032 	WRITE_ONCE(sch->slice_dfl, READ_ONCE(scx_slice_bypass_us) * NSEC_PER_USEC);
7033 	sch->bypass_timestamp = ktime_get_ns();
7034 	scx_add_event(sch, SCX_EV_BYPASS_ACTIVATE, 1);
7035 	return true;
7036 }
7037 
7038 static bool dec_bypass_depth(struct scx_sched *sch)
7039 {
7040 	lockdep_assert_held(&scx_bypass_lock);
7041 
7042 	WARN_ON_ONCE(sch->bypass_depth < 1);
7043 	WRITE_ONCE(sch->bypass_depth, sch->bypass_depth - 1);
7044 	if (sch->bypass_depth != 0)
7045 		return false;
7046 
7047 	WRITE_ONCE(sch->slice_dfl, SCX_SLICE_DFL);
7048 	scx_add_event(sch, SCX_EV_BYPASS_DURATION,
7049 		      ktime_get_ns() - sch->bypass_timestamp);
7050 	return true;
7051 }
7052 
7053 static void enable_bypass_dsp(struct scx_sched *sch)
7054 {
7055 	struct scx_sched *host = scx_parent(sch) ?: sch;
7056 	u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us);
7057 	s32 ret;
7058 
7059 	/*
7060 	 * @sch->bypass_depth transitioning from 0 to 1 triggers enabling.
7061 	 * Shouldn't stagger.
7062 	 */
7063 	if (WARN_ON_ONCE(test_and_set_bit(0, &sch->bypass_dsp_claim)))
7064 		return;
7065 
7066 	/*
7067 	 * When a sub-sched bypasses, its tasks are queued on the bypass DSQs of
7068 	 * the nearest non-bypassing ancestor or root. The parent hosts them
7069 	 * unless it is bypassing too. Then they sit further up and the parent
7070 	 * consumes only its own bypass DSQs, empty until it leaves bypass and
7071 	 * becomes the host. Either way the nearest non-bypassing ancestor or
7072 	 * root has bypass dispatch enabled while a descendant is bypassing,
7073 	 * which is all that's required.
7074 	 *
7075 	 * scx_bypass_dsp_enabled() test is used to determine whether to enter
7076 	 * the bypass dispatch handling path from both bypassing and hosting
7077 	 * scheds. Bump enable depth on both @sch and bypass dispatch host.
7078 	 */
7079 	ret = atomic_inc_return(&sch->bypass_dsp_enable_depth);
7080 	WARN_ON_ONCE(ret <= 0);
7081 
7082 	if (host != sch) {
7083 		ret = atomic_inc_return(&host->bypass_dsp_enable_depth);
7084 		WARN_ON_ONCE(ret <= 0);
7085 	}
7086 
7087 	/*
7088 	 * The LB timer will stop running if bypass dispatch is disabled. Start
7089 	 * after enabling bypass dispatch. A kick from a BASE loss can have set
7090 	 * the timer to expire now in the meantime, see scx_bypass_lb_kick().
7091 	 * timer_reduce() keeps that expiry.
7092 	 */
7093 	if (intv_us)
7094 		timer_reduce(&host->bypass_lb_timer, jiffies + usecs_to_jiffies(intv_us));
7095 }
7096 
7097 /* may be called without holding scx_bypass_lock */
7098 void scx_disable_bypass_dsp(struct scx_sched *sch)
7099 {
7100 	s32 ret;
7101 
7102 	if (!test_and_clear_bit(0, &sch->bypass_dsp_claim))
7103 		return;
7104 
7105 	ret = atomic_dec_return(&sch->bypass_dsp_enable_depth);
7106 	WARN_ON_ONCE(ret < 0);
7107 
7108 	if (scx_parent(sch)) {
7109 		ret = atomic_dec_return(&scx_parent(sch)->bypass_dsp_enable_depth);
7110 		WARN_ON_ONCE(ret < 0);
7111 	}
7112 }
7113 
7114 /**
7115  * unbypass_renotify_idle - Arm an idle re-notify for a sched leaving bypass
7116  * @rq: rq of the cpu leaving bypass
7117  * @pos: scheduler that just left bypass on @rq's cpu
7118  * @pcpu: @pos's per-cpu state for @rq's cpu
7119  *
7120  * A sched leaving bypass is owed the ops.update_idle() calls suppressed while
7121  * bypassing. A cpu that goes idle during the bypass window and stays idle won't
7122  * produce a notification. Arm a re-notify that scx_bypass()'s resched flushes
7123  * on the next idle pick.
7124  *
7125  * An acute case is ops.sub_attach(). If the parent grants the child cids while
7126  * attaching, when attach is complete and bypass is lifted, the child may hold
7127  * idle cids it never saw go idle.
7128  *
7129  * The root is no exception as bypass suppresses its notifications the same way.
7130  * However, the root uses a separate per-rq flag so its re-notify keeps working
7131  * even when !CONFIG_EXT_SUB_SCHED.
7132  */
7133 static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos,
7134 				   struct scx_sched_pcpu *pcpu)
7135 {
7136 	if (!pos->level) {
7137 		rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY;
7138 		return;
7139 	}
7140 #ifdef CONFIG_EXT_SUB_SCHED
7141 	pcpu->idle_renotify = true;
7142 	rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY;
7143 #endif
7144 }
7145 
7146 /**
7147  * scx_bypass - [Un]bypass scx_ops and guarantee forward progress
7148  * @sch: sched to bypass
7149  * @bypass: true for bypass, false for unbypass
7150  *
7151  * Bypassing guarantees that all runnable tasks make forward progress without
7152  * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might
7153  * be held by tasks that the BPF scheduler is forgetting to run, which
7154  * unfortunately also excludes toggling the static branches.
7155  *
7156  * Let's work around by overriding a couple ops and modifying behaviors based on
7157  * the DISABLING state and then cycling the queued tasks through dequeue/enqueue
7158  * to force global FIFO scheduling.
7159  *
7160  * - ops.select_cpu() is ignored and the default select_cpu() is used.
7161  *
7162  * - ops.enqueue() is ignored and tasks are queued in simple global FIFO order.
7163  *   %SCX_OPS_ENQ_LAST is also ignored.
7164  *
7165  * - ops.dispatch() is ignored.
7166  *
7167  * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called.
7168  *   The bypassed tasks are on the bypass DSQs, under rescue or admitted to a
7169  *   local DSQ, out of their reach.
7170  *
7171  * - dispatch_one() expires prev's slice before its keep test as the slice can't
7172  *   be trusted. Each tick does the same and rotates the running task to the
7173  *   tail of the queue. A kernel slice is left alone, see
7174  *   %SCX_TASK_KERNEL_SLICE.
7175  *
7176  * - pick_next_task() suppresses zero slice warning.
7177  *
7178  * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM
7179  *   operations.
7180  *
7181  * - scx_prio_less() reverts to the default runnable_at order.
7182  */
7183 void scx_bypass(struct scx_sched *sch, bool bypass)
7184 {
7185 	struct scx_sched *pos;
7186 	unsigned long flags;
7187 	int cpu;
7188 	bool changed;
7189 
7190 	raw_spin_lock_irqsave(&scx_bypass_lock, flags);
7191 
7192 	if (bypass) {
7193 		changed = inc_bypass_depth(sch);
7194 		if (changed)
7195 			enable_bypass_dsp(sch);
7196 	} else {
7197 		changed = dec_bypass_depth(sch);
7198 	}
7199 
7200 	/*
7201 	 * Bypass state is propagated to all descendants - an scx_sched bypasses
7202 	 * if itself or any of its ancestors are in bypass mode. A descendant's
7203 	 * depth is the sum of its own bypasses and its ancestors', so every
7204 	 * call propagates whether or not @sch changed state. The descendants'
7205 	 * claims are released below with @sch's, after their tasks have moved.
7206 	 */
7207 	raw_spin_lock(&scx_sched_lock);
7208 	scx_for_each_descendant_pre(pos, sch) {
7209 		if (pos == sch)
7210 			continue;
7211 		if (bypass) {
7212 			if (inc_bypass_depth(pos))
7213 				enable_bypass_dsp(pos);
7214 		} else {
7215 			dec_bypass_depth(pos);
7216 		}
7217 	}
7218 	raw_spin_unlock(&scx_sched_lock);
7219 
7220 	/*
7221 	 * A descendant's depth is at least @sch's, so no bypass state changed
7222 	 * unless @sch's did.
7223 	 */
7224 	if (!changed)
7225 		goto unlock;
7226 
7227 	/*
7228 	 * No task property is changing. We just need to make sure all currently
7229 	 * queued tasks are re-queued according to the new scx_bypassing()
7230 	 * state. As an optimization, walk each rq's runnable_list instead of
7231 	 * the scx_tasks list.
7232 	 *
7233 	 * This function can't trust the scheduler and thus can't use
7234 	 * cpus_read_lock(). Walk all possible CPUs instead of online.
7235 	 */
7236 	for_each_possible_cpu(cpu) {
7237 		struct rq *rq = cpu_rq(cpu);
7238 		struct task_struct *p, *n;
7239 
7240 		raw_spin_rq_lock(rq);
7241 		raw_spin_lock(&scx_sched_lock);
7242 
7243 		scx_for_each_descendant_pre(pos, sch) {
7244 			struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
7245 			bool was_bypassing = pcpu->flags & SCX_SCHED_PCPU_BYPASSING;
7246 
7247 			if (pos->bypass_depth) {
7248 				pcpu->flags |= SCX_SCHED_PCPU_BYPASSING;
7249 			} else {
7250 				pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING;
7251 				if (was_bypassing)
7252 					unbypass_renotify_idle(rq, pos, pcpu);
7253 			}
7254 		}
7255 
7256 		raw_spin_unlock(&scx_sched_lock);
7257 
7258 		/*
7259 		 * We need to guarantee that no tasks are on the BPF scheduler
7260 		 * while bypassing. Either we see enabled or the enable path
7261 		 * sees scx_bypassing() before moving tasks to SCX.
7262 		 */
7263 		if (!scx_enabled()) {
7264 			scx_rq_lock_drop(rq);
7265 			raw_spin_rq_unlock(rq);
7266 			continue;
7267 		}
7268 
7269 		/*
7270 		 * Bypass trumps protection. Cycling the queued tasks below
7271 		 * clears theirs. The current donor isn't on the runnable list,
7272 		 * so strip its protection here.
7273 		 *
7274 		 * When bypass lifts, the current donor keeps running on its
7275 		 * slice without a cap check. Evict it if it can't stay on the
7276 		 * cpu.
7277 		 */
7278 		p = rq->donor;
7279 		if (p->sched_class == &ext_sched_class &&
7280 		    scx_is_descendant(scx_task_sched(p), sch)) {
7281 			if (bypass)
7282 				scx_task_slice_ended(rq, p);
7283 			else if (!scx_task_can_stay_on_cpu(rq, p))
7284 				expire_task_slice(p);
7285 		}
7286 
7287 		/*
7288 		 * The use of list_for_each_entry_safe_reverse() is required
7289 		 * because each task is going to be removed from and added back
7290 		 * to the runnable_list during iteration. Because they're added
7291 		 * to the tail of the list, safe reverse iteration can still
7292 		 * visit all nodes.
7293 		 */
7294 		list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list,
7295 						 scx.runnable_node) {
7296 			struct scx_sched *owner = scx_task_sched(p);
7297 
7298 			if (!scx_is_descendant(owner, sch))
7299 				continue;
7300 
7301 			/* cycling deq/enq is enough, see the function comment */
7302 			scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
7303 				/*
7304 				 * The requeue can change the sched running @p,
7305 				 * and the watchdog counts @p's wait against
7306 				 * that sched. Restart the wait unless @owner is
7307 				 * the root, which runs its own tasks bypassing
7308 				 * or not.
7309 				 *
7310 				 * A dispatch in flight writes p->scx.flags
7311 				 * without @p's rq lock. The dequeue waits it
7312 				 * out, so set the flag after it.
7313 				 */
7314 				if (scx_parent(owner))
7315 					p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
7316 			}
7317 		}
7318 
7319 		/* resched to restore ticks and idle state */
7320 		if (cpu_online(cpu) || cpu == smp_processor_id())
7321 			resched_curr(rq);
7322 
7323 		scx_rq_lock_drop(rq);
7324 		raw_spin_rq_unlock(rq);
7325 	}
7326 
7327 	/* disarming must come after moving all tasks out of the bypass DSQs */
7328 	if (!bypass) {
7329 		guard(raw_spinlock)(&scx_sched_lock);
7330 		scx_for_each_descendant_pre(pos, sch)
7331 			if (!pos->bypass_depth)
7332 				scx_disable_bypass_dsp(pos);
7333 	}
7334 unlock:
7335 	raw_spin_unlock_irqrestore(&scx_bypass_lock, flags);
7336 }
7337 
7338 static void free_exit_info(struct scx_exit_info *ei)
7339 {
7340 	kvfree(ei->dump);
7341 	kfree(ei->msg);
7342 	kfree(ei->bt);
7343 	kfree(ei);
7344 }
7345 
7346 static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len)
7347 {
7348 	struct scx_exit_info *ei;
7349 
7350 	ei = kzalloc_obj(*ei);
7351 	if (!ei)
7352 		return NULL;
7353 
7354 	ei->exit_cpu = -1;
7355 	ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN);
7356 	ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL);
7357 	ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL);
7358 
7359 	if (!ei->bt || !ei->msg || !ei->dump) {
7360 		free_exit_info(ei);
7361 		return NULL;
7362 	}
7363 
7364 	return ei;
7365 }
7366 
7367 static const char *scx_exit_reason(enum scx_exit_kind kind)
7368 {
7369 	switch (kind) {
7370 	case SCX_EXIT_UNREG:
7371 		return "unregistered from user space";
7372 	case SCX_EXIT_UNREG_BPF:
7373 		return "unregistered from BPF";
7374 	case SCX_EXIT_UNREG_KERN:
7375 		return "unregistered from the main kernel";
7376 	case SCX_EXIT_SYSRQ:
7377 		return "disabled by sysrq-S";
7378 	case SCX_EXIT_PARENT:
7379 		return "parent exiting";
7380 	case SCX_EXIT_PARENT_KILL:
7381 		return "killed by parent scheduler";
7382 	case SCX_EXIT_ERROR:
7383 		return "runtime error";
7384 	case SCX_EXIT_ERROR_BPF:
7385 		return "scx_bpf_error";
7386 	case SCX_EXIT_ERROR_STALL:
7387 		return "runnable task stall";
7388 	case SCX_EXIT_ERROR_REENQ:
7389 		return "reenqueue limit";
7390 	case SCX_EXIT_ERROR_RESCUE:
7391 		return "rescue bandwidth overload";
7392 	default:
7393 		return "<UNKNOWN>";
7394 	}
7395 }
7396 
7397 static void free_kick_syncs(void)
7398 {
7399 	int cpu;
7400 
7401 	for_each_possible_cpu(cpu) {
7402 		struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
7403 		struct scx_kick_syncs *to_free;
7404 
7405 		/* flush the pending kick before freeing @ksyncs */
7406 		irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
7407 		to_free = rcu_replace_pointer(*ksyncs, NULL, true);
7408 		if (to_free)
7409 			kvfree_rcu(to_free, rcu);
7410 	}
7411 }
7412 
7413 static void refresh_watchdog(void)
7414 {
7415 	struct scx_sched *sch;
7416 	unsigned long intv = ULONG_MAX;
7417 
7418 	/* take the shortest timeout and use its half for watchdog interval */
7419 	rcu_read_lock();
7420 	list_for_each_entry_rcu(sch, &scx_sched_all, all)
7421 		intv = max(min(intv, sch->watchdog_timeout / 2), 1);
7422 	rcu_read_unlock();
7423 
7424 	WRITE_ONCE(scx_watchdog_timestamp, jiffies);
7425 	WRITE_ONCE(scx_watchdog_interval, intv);
7426 
7427 	if (intv < ULONG_MAX)
7428 		mod_delayed_work(system_dfl_wq, &scx_watchdog_work, intv);
7429 	else
7430 		cancel_delayed_work_sync(&scx_watchdog_work);
7431 }
7432 
7433 s32 scx_link_sched(struct scx_sched *sch)
7434 {
7435 	scoped_guard(raw_spinlock_irqsave, &scx_bypass_lock)	/* for the parent bypass check */
7436 	scoped_guard(raw_spinlock, &scx_sched_lock) {
7437 #ifdef CONFIG_EXT_SUB_SCHED
7438 		struct scx_sched *parent = scx_parent(sch);
7439 
7440 		if (parent) {
7441 			s32 ret;
7442 
7443 			/*
7444 			 * Bypass state is spread across per-cpu flags and a
7445 			 * depth count, so inheriting it is tricky and has no
7446 			 * valid use case. Refuse it.
7447 			 */
7448 			if (READ_ONCE(parent->bypass_depth)) {
7449 				scx_error(sch, "parent bypassing (%d)", -EBUSY);
7450 				return -EBUSY;
7451 			}
7452 
7453 			ret = rhashtable_lookup_insert_fast(&scx_sched_hash,
7454 					&sch->hash_node, scx_sched_hash_params);
7455 			if (ret) {
7456 				scx_error(sch, "failed to insert into scx_sched_hash (%d)",
7457 					  ret);
7458 				return ret;
7459 			}
7460 
7461 			list_add_tail_rcu(&sch->sibling, &parent->children);
7462 
7463 			/*
7464 			 * Pairs with the mb after the ->aborting assertion in
7465 			 * scx_claim_exit(). Either we see ->aborting and back
7466 			 * out, or the exit path sees us and exits us.
7467 			 */
7468 			smp_mb();
7469 			if (unlikely(READ_ONCE(parent->aborting))) {
7470 				rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node,
7471 						       scx_sched_hash_params);
7472 				list_del_rcu(&sch->sibling);
7473 				scx_error(sch, "parent disabled (%d)", -ENOENT);
7474 				return -ENOENT;
7475 			}
7476 
7477 			sch->linked = true;
7478 		}
7479 #endif	/* CONFIG_EXT_SUB_SCHED */
7480 
7481 		list_add_tail_rcu(&sch->all, &scx_sched_all);
7482 	}
7483 
7484 	refresh_watchdog();
7485 	return 0;
7486 }
7487 
7488 void scx_unlink_sched(struct scx_sched *sch)
7489 {
7490 	scoped_guard(raw_spinlock_irq, &scx_sched_lock) {
7491 #ifdef CONFIG_EXT_SUB_SCHED
7492 		if (sch->linked) {
7493 			rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node,
7494 					       scx_sched_hash_params);
7495 			list_del_rcu(&sch->sibling);
7496 			sch->linked = false;
7497 		}
7498 #endif	/* CONFIG_EXT_SUB_SCHED */
7499 		list_del_rcu(&sch->all);
7500 	}
7501 
7502 	refresh_watchdog();
7503 }
7504 
7505 /*
7506  * Called to disable future dumps and wait for in-progress one while disabling
7507  * @sch. Once @sch becomes empty during disable, there's no point in dumping it.
7508  * This prevents calling dump ops on a dead sch.
7509  */
7510 void scx_disable_dump(struct scx_sched *sch)
7511 {
7512 	guard(raw_spinlock_irqsave)(&scx_dump_lock);
7513 	sch->dump_disabled = true;
7514 }
7515 
7516 void scx_log_sched_disable(struct scx_sched *sch)
7517 {
7518 	struct scx_exit_info *ei = sch->exit_info;
7519 	const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler";
7520 
7521 	if (ei->kind >= SCX_EXIT_ERROR) {
7522 		pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type,
7523 		       sch->ops.name, ei->reason);
7524 
7525 		if (ei->msg[0] != '\0')
7526 			pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg);
7527 #ifdef CONFIG_STACKTRACE
7528 		stack_trace_print(ei->bt, ei->bt_len, 2);
7529 #endif
7530 	} else {
7531 		pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type,
7532 			sch->ops.name, ei->reason);
7533 	}
7534 }
7535 
7536 static void scx_root_disable(struct scx_sched *sch)
7537 {
7538 	struct scx_task_iter sti;
7539 	struct task_struct *p;
7540 	bool was_switched_all;
7541 	int cpu;
7542 
7543 	/* guarantee forward progress and wait for descendants to be disabled */
7544 	scx_bypass(sch, true);
7545 	drain_descendants(sch);
7546 
7547 	switch (scx_set_enable_state(SCX_DISABLING)) {
7548 	case SCX_DISABLING:
7549 		WARN_ONCE(true, "sched_ext: duplicate disabling instance?");
7550 		break;
7551 	case SCX_DISABLED:
7552 		pr_warn("sched_ext: ops error detected without ops (%s)\n",
7553 			sch->exit_info->msg);
7554 		WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING);
7555 		goto done;
7556 	default:
7557 		break;
7558 	}
7559 
7560 	/*
7561 	 * Here, every runnable task is guaranteed to make forward progress and
7562 	 * we can safely use blocking synchronization constructs. Actually
7563 	 * disable ops.
7564 	 */
7565 	mutex_lock(&scx_enable_mutex);
7566 
7567 	was_switched_all = scx_switched_all();
7568 
7569 	static_branch_disable(&__scx_switched_all);
7570 	WRITE_ONCE(scx_switching_all, false);
7571 
7572 	/*
7573 	 * Shut down cgroup support before tasks so that the cgroup attach and
7574 	 * migration paths don't race against scx_disable_and_exit_task().
7575 	 */
7576 	scx_cgroup_lock();
7577 	scx_cgroup_enabled = false;
7578 	scx_cgroup_exit(sch);
7579 	scx_cgroup_unlock();
7580 
7581 	/*
7582 	 * The BPF scheduler is going away. All tasks including %TASK_DEAD ones
7583 	 * must be switched out and exited synchronously.
7584 	 */
7585 	percpu_down_write(&scx_fork_rwsem);
7586 
7587 	scx_init_task_enabled = false;
7588 
7589 	scx_task_iter_start(&sti, NULL);
7590 	while ((p = scx_task_iter_next_locked(&sti))) {
7591 		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
7592 		const struct sched_class *old_class = p->sched_class;
7593 		const struct sched_class *new_class = scx_setscheduler_class(p);
7594 
7595 		update_rq_clock(task_rq(p));
7596 
7597 		if (old_class != new_class)
7598 			queue_flags |= DEQUEUE_CLASS;
7599 
7600 		scoped_guard (sched_change, p, queue_flags) {
7601 			p->sched_class = new_class;
7602 		}
7603 
7604 		scx_disable_and_exit_task(scx_task_sched(p), p);
7605 	}
7606 	scx_task_iter_stop(&sti);
7607 
7608 	scx_disable_dump(sch);
7609 
7610 	scx_cgroup_lock();
7611 	set_cgroup_sched(sch_cgroup(sch), NULL);
7612 	scx_cgroup_unlock();
7613 
7614 	percpu_up_write(&scx_fork_rwsem);
7615 
7616 	/*
7617 	 * Re-balance the dl_server bandwidth reservations: detach ext_server
7618 	 * (no more sched_ext tasks) and reinstate fair_server if it was
7619 	 * previously detached because we were running in full mode.
7620 	 *
7621 	 * Unlike the enable path, this runs on a recovery path that cannot
7622 	 * fail, so we use dl_server_swap_bw() to atomically free ext_server's
7623 	 * bandwidth and reclaim it for fair_server under the same dl_b lock.
7624 	 *
7625 	 * The swap can still fail with -EBUSY if someone bumped ext_server's
7626 	 * runtime via debugfs between enable and disable; in that narrow case
7627 	 * both servers end up detached and we just WARN.
7628 	 */
7629 	for_each_possible_cpu(cpu) {
7630 		struct rq *rq = cpu_rq(cpu);
7631 
7632 		scoped_guard(rq_lock_irqsave, rq) {
7633 			update_rq_clock(rq);
7634 			if (was_switched_all) {
7635 				if (WARN_ON_ONCE(dl_server_swap_bw(&rq->ext_server,
7636 								   &rq->fair_server)))
7637 					pr_warn("failed to re-attach fair_server on CPU %d\n", cpu);
7638 			} else {
7639 				dl_server_detach_bw(&rq->ext_server);
7640 			}
7641 		}
7642 	}
7643 
7644 	scx_ops_cid_sched_updated_disable(sch);
7645 
7646 	/* no task is on scx, turn off all the switches and flush in-progress calls */
7647 	static_branch_disable(&__scx_enabled);
7648 	static_branch_disable(&__scx_is_cid_type);
7649 	if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
7650 		static_branch_disable(&__scx_tid_to_task_enabled);
7651 	bitmap_zero(sch->has_op, SCX_OPI_END);
7652 	scx_idle_disable();
7653 	synchronize_rcu();
7654 	if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
7655 		rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
7656 
7657 	scx_log_sched_disable(sch);
7658 
7659 	if (sch->ops.exit)
7660 		SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
7661 
7662 	/*
7663 	 * @sch's non-ops programs such as timers and tracers can fire after
7664 	 * ops.exit(). Now that exit is complete, stop scx_prog_sched() from
7665 	 * resolving to @sch and drain in-flight resolvers.
7666 	 */
7667 	WRITE_ONCE(sch->dead, true);
7668 	synchronize_rcu();
7669 
7670 	scx_unlink_sched(sch);
7671 
7672 	/*
7673 	 * scx_root clearing and cid table retirement must be inside
7674 	 * cpus_read_lock(). See handle_hotplug().
7675 	 */
7676 	cpus_read_lock();
7677 	RCU_INIT_POINTER(scx_root, NULL);
7678 	scx_cid_retire_tables();
7679 	cpus_read_unlock();
7680 
7681 	/*
7682 	 * Delete the kobject from the hierarchy synchronously. Otherwise, sysfs
7683 	 * could observe an object of the same name still in the hierarchy when
7684 	 * the next scheduler is loaded.
7685 	 */
7686 #ifdef CONFIG_EXT_SUB_SCHED
7687 	if (sch->sub_kset)
7688 		kobject_del(&sch->sub_kset->kobj);
7689 #endif
7690 	/* not added if enable failed before scx_sched_sysfs_add() */
7691 	if (sch->kobj.state_in_sysfs)
7692 		kobject_del(&sch->kobj);
7693 
7694 	free_kick_syncs();
7695 
7696 	mutex_unlock(&scx_enable_mutex);
7697 
7698 	WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING);
7699 done:
7700 	scx_bypass(sch, false);
7701 }
7702 
7703 /**
7704  * scx_propagate_exit_irq_workfn - Claim SCX_EXIT_PARENT on the exiting subtree
7705  * @irq_work: &scx_sched.propagate_exit_irq_work
7706  *
7707  * Queued by scx_claim_exit() after a non-PARENT claim. Claims SCX_EXIT_PARENT
7708  * on each descendant, giving every one its own disable work - most of disabling
7709  * is serialized but ops.exit() can take arbitrarily long and running them in
7710  * separate helper kthreads parallelizes it. No recursion as only non-PARENT
7711  * claims propagate.
7712  */
7713 static void scx_propagate_exit_irq_workfn(struct irq_work *irq_work)
7714 {
7715 	struct scx_sched *sch = container_of(irq_work, struct scx_sched,
7716 					     propagate_exit_irq_work);
7717 	struct scx_sched *pos;
7718 
7719 	scoped_guard (raw_spinlock_irqsave, &scx_sched_lock) {
7720 		scx_for_each_descendant_pre(pos, sch)
7721 			scx_disable(pos, SCX_EXIT_PARENT);
7722 	}
7723 }
7724 
7725 /*
7726  * Claim the exit on @sch. The caller must ensure that the helper kthread work
7727  * is kicked before the current task can be preempted. Once exit_kind is
7728  * claimed, scx_error() can no longer trigger, so if the current task gets
7729  * preempted and the BPF scheduler fails to schedule it back, the helper work
7730  * will never be kicked and the whole system can wedge.
7731  *
7732  * Lock-free and safe to call from any context including NMI.
7733  */
7734 static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind)
7735 {
7736 	int none = SCX_EXIT_NONE;
7737 
7738 	lockdep_assert_preemption_disabled();
7739 
7740 	if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE))
7741 		kind = SCX_EXIT_ERROR;
7742 
7743 	if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind))
7744 		return false;
7745 
7746 	if (kind == SCX_EXIT_PARENT) {
7747 		/* an ancestor is already sweeping the subtree */
7748 		WRITE_ONCE(sch->aborting, true);
7749 	} else {
7750 		struct scx_sched *pos;
7751 
7752 		/*
7753 		 * CPUs may be live-locked in the dispatch paths of @sch or its
7754 		 * descendants, which ->aborting breaks. Sweep the subtree
7755 		 * locklessly so that this works from NMI. smp_store_mb() orders
7756 		 * each node's ->aborting store before its children are walked -
7757 		 * either we see a racing scx_link_sched() on ->children or it
7758 		 * sees ->aborting.
7759 		 */
7760 		scoped_guard (rcu) {
7761 			scx_for_each_descendant_pre(pos, sch)
7762 				smp_store_mb(pos->aborting, true);
7763 		}
7764 
7765 		irq_work_queue(&sch->propagate_exit_irq_work);
7766 	}
7767 
7768 	/* fired after ->aborting is set so callbacks can't delay recovery */
7769 	trace_sched_ext_exit(sch, kind);
7770 
7771 	return true;
7772 }
7773 
7774 static void scx_disable_workfn(struct kthread_work *work)
7775 {
7776 	struct scx_sched *sch = container_of(work, struct scx_sched, disable_work);
7777 	struct scx_exit_info *ei = sch->exit_info;
7778 	int kind;
7779 
7780 	kind = atomic_read(&sch->exit_kind);
7781 	while (true) {
7782 		if (kind == SCX_EXIT_DONE)	/* already disabled? */
7783 			return;
7784 		WARN_ON_ONCE(kind == SCX_EXIT_NONE);
7785 		if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE))
7786 			break;
7787 	}
7788 	ei->kind = kind;
7789 	ei->reason = scx_exit_reason(ei->kind);
7790 
7791 	if (scx_parent(sch))
7792 		scx_sub_disable(sch);
7793 	else
7794 		scx_root_disable(sch);
7795 }
7796 
7797 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind)
7798 {
7799 	guard(preempt)();
7800 	if (scx_claim_exit(sch, kind))
7801 		irq_work_queue(&sch->disable_irq_work);
7802 }
7803 
7804 /**
7805  * scx_flush_disable_work - flush the disable work and wait for it to finish
7806  * @sch: the scheduler
7807  *
7808  * sch->disable_work might still not queued, causing kthread_flush_work()
7809  * as a noop. Syncing the irq_work first is required to guarantee the
7810  * kthread work has been queued before waiting for it.
7811  */
7812 void scx_flush_disable_work(struct scx_sched *sch)
7813 {
7814 	int kind;
7815 
7816 	/*
7817 	 * scx_disable_workfn() sets SCX_EXIT_DONE before tearing down. Read
7818 	 * exit_kind with acquire ordering (paired with atomic_try_cmpxchg() in
7819 	 * scx_disable_workfn()) before syncing and flushing so that, once DONE
7820 	 * is seen, the following flush observes the queued work and waits for
7821 	 * the teardown to finish.
7822 	 */
7823 	do {
7824 		kind = atomic_read_acquire(&sch->exit_kind);
7825 		irq_work_sync(&sch->disable_irq_work);
7826 		kthread_flush_work(&sch->disable_work);
7827 	} while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);
7828 }
7829 
7830 static void dump_newline(struct seq_buf *s)
7831 {
7832 	trace_sched_ext_dump("");
7833 
7834 	/* @s may be zero sized and seq_buf triggers WARN if so */
7835 	if (s->size)
7836 		seq_buf_putc(s, '\n');
7837 }
7838 
7839 __printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...)
7840 {
7841 	va_list args;
7842 
7843 #ifdef CONFIG_TRACEPOINTS
7844 	if (trace_sched_ext_dump_enabled()) {
7845 		/* protected by scx_dump_lock */
7846 		static char line_buf[SCX_EXIT_MSG_LEN];
7847 
7848 		va_start(args, fmt);
7849 		vscnprintf(line_buf, sizeof(line_buf), fmt, args);
7850 		va_end(args);
7851 
7852 		trace_call__sched_ext_dump(line_buf);
7853 	}
7854 #endif
7855 	/* @s may be zero sized and seq_buf triggers WARN if so */
7856 	if (s->size) {
7857 		va_start(args, fmt);
7858 		seq_buf_vprintf(s, fmt, args);
7859 		va_end(args);
7860 
7861 		seq_buf_putc(s, '\n');
7862 	}
7863 }
7864 
7865 static void dump_stack_trace(struct seq_buf *s, const char *prefix,
7866 			     const unsigned long *bt, unsigned int len)
7867 {
7868 	unsigned int i;
7869 
7870 	for (i = 0; i < len; i++)
7871 		scx_dump_line(s, "%s%pS", prefix, (void *)bt[i]);
7872 }
7873 
7874 static void ops_dump_init(struct seq_buf *s, const char *prefix)
7875 {
7876 	struct scx_dump_data *dd = &scx_dump_data;
7877 
7878 	lockdep_assert_irqs_disabled();
7879 
7880 	dd->cpu = smp_processor_id();		/* allow scx_bpf_dump() */
7881 	dd->first = true;
7882 	dd->cursor = 0;
7883 	dd->s = s;
7884 	dd->prefix = prefix;
7885 }
7886 
7887 static void ops_dump_flush(void)
7888 {
7889 	struct scx_dump_data *dd = &scx_dump_data;
7890 	char *line = dd->buf.line;
7891 
7892 	if (!dd->cursor)
7893 		return;
7894 
7895 	/*
7896 	 * There's something to flush and this is the first line. Insert a blank
7897 	 * line to distinguish ops dump.
7898 	 */
7899 	if (dd->first) {
7900 		dump_newline(dd->s);
7901 		dd->first = false;
7902 	}
7903 
7904 	/*
7905 	 * There may be multiple lines in $line. Scan and emit each line
7906 	 * separately.
7907 	 */
7908 	while (true) {
7909 		char *end = line;
7910 		char c;
7911 
7912 		while (*end != '\n' && *end != '\0')
7913 			end++;
7914 
7915 		/*
7916 		 * If $line overflowed, it may not have newline at the end.
7917 		 * Always emit with a newline.
7918 		 */
7919 		c = *end;
7920 		*end = '\0';
7921 		scx_dump_line(dd->s, "%s%s", dd->prefix, line);
7922 		if (c == '\0')
7923 			break;
7924 
7925 		/* move to the next line */
7926 		end++;
7927 		if (*end == '\0')
7928 			break;
7929 		line = end;
7930 	}
7931 
7932 	dd->cursor = 0;
7933 }
7934 
7935 static void ops_dump_exit(void)
7936 {
7937 	ops_dump_flush();
7938 	scx_dump_data.cpu = -1;
7939 }
7940 
7941 static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_dump_ctx *dctx,
7942 			  struct rq *rq, struct task_struct *p, char marker)
7943 {
7944 	static unsigned long bt[SCX_EXIT_BT_LEN];
7945 	struct scx_sched *task_sch = scx_task_sched(p);
7946 	const char *own_marker;
7947 	char sch_id_buf[32];
7948 	char dsq_id_buf[19] = "(n/a)";
7949 	unsigned long ops_state = atomic_long_read(&p->scx.ops_state);
7950 	unsigned int bt_len = 0;
7951 
7952 	own_marker = task_sch == sch ? "*" : "";
7953 
7954 	if (task_sch->level == 0)
7955 		scnprintf(sch_id_buf, sizeof(sch_id_buf), "root");
7956 	else
7957 		scnprintf(sch_id_buf, sizeof(sch_id_buf), "sub%d-%llu",
7958 			  task_sch->level, task_sch->ops.sub_cgroup_id);
7959 
7960 	if (p->scx.dsq)
7961 		scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx",
7962 			  (unsigned long long)p->scx.dsq->id);
7963 
7964 	dump_newline(s);
7965 	scx_dump_line(s, " %c%c %s[%d] %s%s %+ldms",
7966 		      marker, task_state_to_char(p), p->comm, p->pid, own_marker, sch_id_buf,
7967 		      jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies));
7968 	scx_dump_line(s, "      scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu",
7969 		      scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT,
7970 		      p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags,
7971 		      ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT);
7972 	scx_dump_line(s, "      sticky/holding_cpu=%d/%d dsq_id=%s",
7973 		      p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf);
7974 	scx_dump_line(s, "      dsq_vtime=%llu slice=%llu weight=%u",
7975 		      p->scx.dsq_vtime, p->scx.slice, p->scx.weight);
7976 	scx_dump_line(s, "      cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr),
7977 		      p->migration_disabled);
7978 
7979 	if (SCX_HAS_OP(sch, dump_task)) {
7980 		ops_dump_init(s, "    ");
7981 		SCX_CALL_OP(sch, dump_task, rq, dctx, p);
7982 		ops_dump_exit();
7983 	}
7984 
7985 #ifdef CONFIG_STACKTRACE
7986 	bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1);
7987 #endif
7988 	if (bt_len) {
7989 		dump_newline(s);
7990 		dump_stack_trace(s, "    ", bt, bt_len);
7991 	}
7992 }
7993 
7994 static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
7995 			 struct scx_dump_ctx *dctx, int cpu,
7996 			 bool dump_all_tasks)
7997 {
7998 	struct rq *rq = cpu_rq(cpu);
7999 	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
8000 	struct rq_flags rf;
8001 	struct task_struct *p;
8002 	struct seq_buf ns;
8003 	size_t avail, used;
8004 	char *buf;
8005 	bool idle;
8006 
8007 	rq_lock_irqsave(rq, &rf);
8008 
8009 	idle = list_empty(&rq->scx.runnable_list) &&
8010 		rq->curr->sched_class == &idle_sched_class;
8011 
8012 	if (idle && !SCX_HAS_OP(sch, dump_cpu))
8013 		goto next;
8014 
8015 	/*
8016 	 * We don't yet know whether ops.dump_cpu() will produce output
8017 	 * and we may want to skip the default CPU dump if it doesn't.
8018 	 * Use a nested seq_buf to generate the standard dump so that we
8019 	 * can decide whether to commit later.
8020 	 */
8021 	avail = seq_buf_get_buf(s, &buf);
8022 	seq_buf_init(&ns, buf, avail);
8023 
8024 	dump_newline(&ns);
8025 	scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu",
8026 		      cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released,
8027 		      rq->scx.kick_sync);
8028 	scx_rescue_dump(&ns, rq);
8029 	scx_dump_line(&ns, "          curr=%s[%d] class=%ps",
8030 		      rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
8031 	scx_dump_line(&ns, "          donor=%s[%d] class=%ps",
8032 		      rq->donor->comm, rq->donor->pid, rq->donor->sched_class);
8033 	if (!cpumask_empty(pcpu->cpus_to_kick))
8034 		scx_dump_line(&ns, "  cpus_to_kick   : %*pb",
8035 			      cpumask_pr_args(pcpu->cpus_to_kick));
8036 	if (!cpumask_empty(pcpu->cpus_to_kick_if_idle))
8037 		scx_dump_line(&ns, "  idle_to_kick   : %*pb",
8038 			      cpumask_pr_args(pcpu->cpus_to_kick_if_idle));
8039 	if (!cpumask_empty(pcpu->cpus_to_preempt))
8040 		scx_dump_line(&ns, "  cpus_to_preempt: %*pb",
8041 			      cpumask_pr_args(pcpu->cpus_to_preempt));
8042 	if (!cpumask_empty(pcpu->cpus_to_preempt_lazy))
8043 		scx_dump_line(&ns, "  preempt_lazy   : %*pb",
8044 			      cpumask_pr_args(pcpu->cpus_to_preempt_lazy));
8045 	if (!cpumask_empty(pcpu->cpus_to_wait))
8046 		scx_dump_line(&ns, "  cpus_to_wait   : %*pb",
8047 			      cpumask_pr_args(pcpu->cpus_to_wait));
8048 	if (!cpumask_empty(rq->scx.cpus_to_sync))
8049 		scx_dump_line(&ns, "  cpus_to_sync   : %*pb",
8050 			      cpumask_pr_args(rq->scx.cpus_to_sync));
8051 
8052 	used = seq_buf_used(&ns);
8053 	if (SCX_HAS_OP(sch, dump_cpu)) {
8054 		ops_dump_init(&ns, "  ");
8055 		SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle);
8056 		ops_dump_exit();
8057 	}
8058 
8059 	/*
8060 	 * If idle && nothing generated by ops.dump_cpu(), there's
8061 	 * nothing interesting. Skip.
8062 	 */
8063 	if (idle && used == seq_buf_used(&ns))
8064 		goto next;
8065 
8066 	/*
8067 	 * $s may already have overflowed when $ns was created. If so,
8068 	 * calling commit on it will trigger BUG.
8069 	 */
8070 	if (avail) {
8071 		seq_buf_commit(s, seq_buf_used(&ns));
8072 		if (seq_buf_has_overflowed(&ns))
8073 			seq_buf_set_overflow(s);
8074 	}
8075 
8076 	if (rq->curr->sched_class == &ext_sched_class &&
8077 	    (dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
8078 		scx_dump_task(sch, s, dctx, rq, rq->curr, '*');
8079 	if (rq->donor != rq->curr &&
8080 	    rq->donor->sched_class == &ext_sched_class &&
8081 	    (dump_all_tasks || scx_task_on_sched(sch, rq->donor)))
8082 		scx_dump_task(sch, s, dctx, rq, rq->donor, ' ');
8083 
8084 	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
8085 		if (dump_all_tasks || scx_task_on_sched(sch, p))
8086 			scx_dump_task(sch, s, dctx, rq, p, ' ');
8087 next:
8088 	rq_unlock_irqrestore(rq, &rf);
8089 }
8090 
8091 /*
8092  * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless
8093  * of which scheduler they belong to. If false, only dump tasks owned by @sch.
8094  * For SysRq-D dumps, @dump_all_tasks=false since all schedulers are dumped
8095  * separately. For error dumps, @dump_all_tasks=true since only the failing
8096  * scheduler is dumped.
8097  */
8098 static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
8099 			   size_t dump_len, bool dump_all_tasks)
8100 {
8101 	static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n";
8102 	struct scx_dump_ctx dctx = {
8103 		.kind = ei->kind,
8104 		.exit_code = ei->exit_code,
8105 		.reason = ei->reason,
8106 		.at_ns = ktime_get_ns(),
8107 		.at_jiffies = jiffies,
8108 	};
8109 	struct seq_buf s;
8110 	struct scx_event_stats events;
8111 	int cpu;
8112 
8113 	guard(raw_spinlock_irqsave)(&scx_dump_lock);
8114 
8115 	if (sch->dump_disabled)
8116 		return;
8117 
8118 	seq_buf_init(&s, ei->dump, dump_len);
8119 
8120 #ifdef CONFIG_EXT_SUB_SCHED
8121 	if (sch->level == 0)
8122 		scx_dump_line(&s, "%s: root", sch->ops.name);
8123 	else
8124 		scx_dump_line(&s, "%s: sub%d-%llu %s",
8125 			      sch->ops.name, sch->level, sch->ops.sub_cgroup_id,
8126 			      sch->cgrp_path);
8127 #endif
8128 	if (ei->kind == SCX_EXIT_NONE) {
8129 		scx_dump_line(&s, "Debug dump triggered by %s", ei->reason);
8130 	} else {
8131 		if (ei->exit_cpu >= 0)
8132 			scx_dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:",
8133 				      current->comm, current->pid, ei->kind,
8134 				      ei->exit_cpu);
8135 		else
8136 			scx_dump_line(&s, "%s[%d] triggered exit kind %d:",
8137 				      current->comm, current->pid, ei->kind);
8138 		scx_dump_line(&s, "  %s (%s)", ei->reason, ei->msg);
8139 		dump_newline(&s);
8140 		scx_dump_line(&s, "Backtrace:");
8141 		dump_stack_trace(&s, "  ", ei->bt, ei->bt_len);
8142 	}
8143 
8144 	if (SCX_HAS_OP(sch, dump)) {
8145 		ops_dump_init(&s, "");
8146 		SCX_CALL_OP(sch, dump, NULL, &dctx);
8147 		ops_dump_exit();
8148 	}
8149 
8150 	dump_newline(&s);
8151 	scx_dump_line(&s, "CPU states");
8152 	scx_dump_line(&s, "----------");
8153 
8154 	/*
8155 	 * Dump stalled CPUs first so they aren't lost to dump truncation, then
8156 	 * walk the rest in order. Fall back to exit_cpu if no stall mask set.
8157 	 */
8158 	if (!cpumask_empty(sch->stall_cpus)) {
8159 		for_each_cpu(cpu, sch->stall_cpus)
8160 			scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
8161 		for_each_possible_cpu(cpu) {
8162 			if (!cpumask_test_cpu(cpu, sch->stall_cpus))
8163 				scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
8164 		}
8165 	} else {
8166 		if (ei->exit_cpu >= 0)
8167 			scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks);
8168 		for_each_possible_cpu(cpu) {
8169 			if (cpu != ei->exit_cpu)
8170 				scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
8171 		}
8172 	}
8173 
8174 	dump_newline(&s);
8175 	scx_dump_line(&s, "Event counters");
8176 	scx_dump_line(&s, "--------------");
8177 
8178 	scx_read_events(sch, &events);
8179 #define SCX_EVENT(name)	scx_dump_event(s, &events, name)
8180 	SCX_EVENTS_LIST(SCX_EVENT);
8181 #undef SCX_EVENT
8182 
8183 	if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))
8184 		memcpy(ei->dump + dump_len - sizeof(trunc_marker),
8185 		       trunc_marker, sizeof(trunc_marker));
8186 }
8187 
8188 static void scx_disable_irq_workfn(struct irq_work *irq_work)
8189 {
8190 	struct scx_sched *sch = container_of(irq_work, struct scx_sched, disable_irq_work);
8191 	struct scx_exit_info *ei = sch->exit_info;
8192 
8193 	if (ei->kind >= SCX_EXIT_ERROR)
8194 		scx_dump_state(sch, ei, sch->ops.exit_dump_len, true);
8195 
8196 	kthread_queue_work(sch->helper, &sch->disable_work);
8197 }
8198 
8199 /* finish exit_info and kick the disable work, ei->msg must already be set */
8200 static void scx_finish_exit(struct scx_sched *sch, enum scx_exit_kind kind,
8201 			    s64 exit_code, s32 exit_cpu)
8202 {
8203 	struct scx_exit_info *ei = sch->exit_info;
8204 
8205 	ei->exit_code = exit_code;
8206 #ifdef CONFIG_STACKTRACE
8207 	/*
8208 	 * stack_trace_save()'s NMI-safety is arch-dependent and undocumented.
8209 	 * Skip the backtrace when exiting from NMI.
8210 	 */
8211 	if (kind >= SCX_EXIT_ERROR && !in_nmi())
8212 		ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
8213 #endif
8214 	/*
8215 	 * Set ei->kind and ->reason for scx_dump_state(). They'll be set again
8216 	 * in scx_disable_workfn().
8217 	 */
8218 	ei->kind = kind;
8219 	ei->reason = scx_exit_reason(ei->kind);
8220 	ei->exit_cpu = exit_cpu;
8221 
8222 	irq_work_queue(&sch->disable_irq_work);
8223 }
8224 
8225 bool scx_vexit(struct scx_sched *sch,
8226 	       enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu,
8227 	       const char *fmt, va_list args)
8228 {
8229 	struct scx_exit_info *ei = sch->exit_info;
8230 
8231 	guard(preempt)();
8232 
8233 	if (!scx_claim_exit(sch, kind))
8234 		return false;
8235 
8236 	vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args);
8237 
8238 	scx_finish_exit(sch, kind, exit_code, exit_cpu);
8239 	return true;
8240 }
8241 
8242 static int alloc_kick_syncs(void)
8243 {
8244 	int cpu;
8245 
8246 	/*
8247 	 * Allocate per-CPU arrays sized by nr_cpu_ids. Use kvzalloc as size
8248 	 * can exceed percpu allocator limits on large machines.
8249 	 */
8250 	for_each_possible_cpu(cpu) {
8251 		struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
8252 		struct scx_kick_syncs *new_ksyncs;
8253 
8254 		WARN_ON_ONCE(rcu_access_pointer(*ksyncs));
8255 
8256 		new_ksyncs = kvzalloc_node(struct_size(new_ksyncs, syncs, nr_cpu_ids),
8257 					   GFP_KERNEL, cpu_to_node(cpu));
8258 		if (!new_ksyncs) {
8259 			free_kick_syncs();
8260 			return -ENOMEM;
8261 		}
8262 
8263 		rcu_assign_pointer(*ksyncs, new_ksyncs);
8264 	}
8265 
8266 	return 0;
8267 }
8268 
8269 static void free_pnode(struct scx_sched_pnode *pnode)
8270 {
8271 	if (!pnode)
8272 		return;
8273 	exit_dsq(&pnode->global_dsq);
8274 	kfree(pnode);
8275 }
8276 
8277 static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node)
8278 {
8279 	struct scx_sched_pnode *pnode;
8280 
8281 	pnode = kzalloc_node(sizeof(*pnode), GFP_KERNEL, node);
8282 	if (!pnode)
8283 		return NULL;
8284 
8285 	if (scx_init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) {
8286 		kfree(pnode);
8287 		return NULL;
8288 	}
8289 
8290 	return pnode;
8291 }
8292 
8293 /*
8294  * Allocate and initialize a new scx_sched. @cgrp's reference is always
8295  * consumed whether the function succeeds or fails.
8296  */
8297 struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
8298 					  struct cgroup *cgrp,
8299 					  struct scx_sched *parent)
8300 {
8301 	struct sched_ext_ops *ops = cmd->ops;
8302 	struct scx_sched *sch;
8303 	s32 level = parent ? parent->level + 1 : 0;
8304 	s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids;
8305 
8306 	sch = kzalloc_flex(*sch, ancestors, level + 1);
8307 	if (!sch) {
8308 		ret = -ENOMEM;
8309 		goto err_put_cgrp;
8310 	}
8311 
8312 	sch->exit_info = alloc_exit_info(ops->exit_dump_len);
8313 	if (!sch->exit_info) {
8314 		ret = -ENOMEM;
8315 		goto err_free_sch;
8316 	}
8317 
8318 	ret = rhashtable_init(&sch->dsq_hash, &dsq_hash_params);
8319 	if (ret < 0)
8320 		goto err_free_ei;
8321 
8322 	sch->pnode = kzalloc_objs(sch->pnode[0], nr_node_ids);
8323 	if (!sch->pnode) {
8324 		ret = -ENOMEM;
8325 		goto err_free_hash;
8326 	}
8327 
8328 	for_each_node_state(node, N_POSSIBLE) {
8329 		sch->pnode[node] = alloc_pnode(sch, node);
8330 		if (!sch->pnode[node]) {
8331 			ret = -ENOMEM;
8332 			goto err_free_pnode;
8333 		}
8334 	}
8335 
8336 	sch->dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
8337 	sch->pcpu = __alloc_percpu(struct_size_t(struct scx_sched_pcpu,
8338 						 dsp_ctx.buf, sch->dsp_max_batch),
8339 				   __alignof__(struct scx_sched_pcpu));
8340 	if (!sch->pcpu) {
8341 		ret = -ENOMEM;
8342 		goto err_free_pnode;
8343 	}
8344 
8345 	for_each_possible_cpu(cpu) {
8346 		ret = scx_init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch);
8347 		if (ret) {
8348 			bypass_fail_cpu = cpu;
8349 			goto err_free_pcpu;
8350 		}
8351 	}
8352 
8353 	for_each_possible_cpu(cpu) {
8354 		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
8355 
8356 		node = cpu_to_node(cpu);
8357 		pcpu->sch = sch;
8358 		INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node);
8359 #ifdef CONFIG_EXT_SUB_SCHED
8360 		init_llist_node(&pcpu->ecaps_to_sync_node);
8361 #endif
8362 		INIT_LIST_HEAD(&pcpu->to_kick_node);
8363 		if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) ||
8364 		    !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) ||
8365 		    !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) ||
8366 		    !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) ||
8367 		    !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) {
8368 			ret = -ENOMEM;
8369 			goto err_free_pcpu;
8370 		}
8371 	}
8372 
8373 	sch->helper = kthread_run_worker(0, "sched_ext_helper");
8374 	if (IS_ERR(sch->helper)) {
8375 		ret = PTR_ERR(sch->helper);
8376 		goto err_free_pcpu;
8377 	}
8378 
8379 	sched_set_fifo(sch->helper->task);
8380 
8381 	if (parent)
8382 		memcpy(sch->ancestors, parent->ancestors,
8383 		       level * sizeof(parent->ancestors[0]));
8384 	sch->ancestors[level] = sch;
8385 	sch->level = level;
8386 	sch->id = atomic64_inc_return(&scx_sched_id_cursor);
8387 
8388 	if (ops->timeout_ms)
8389 		sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms);
8390 	else
8391 		sch->watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT;
8392 
8393 	sch->slice_dfl = SCX_SLICE_DFL;
8394 	atomic_set(&sch->exit_kind, SCX_EXIT_NONE);
8395 	sch->disable_irq_work = IRQ_WORK_INIT_HARD(scx_disable_irq_workfn);
8396 	sch->propagate_exit_irq_work = IRQ_WORK_INIT_HARD(scx_propagate_exit_irq_workfn);
8397 	kthread_init_work(&sch->disable_work, scx_disable_workfn);
8398 	timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);
8399 
8400 	if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) {
8401 		ret = -ENOMEM;
8402 		goto err_stop_helper;
8403 	}
8404 	if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) {
8405 		ret = -ENOMEM;
8406 		goto err_free_lb_cpumask;
8407 	}
8408 	if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) {
8409 		ret = -ENOMEM;
8410 		goto err_free_lb_resched_cpumask;
8411 	}
8412 	/*
8413 	 * Copy ops through the right union view. For cid-form the source is
8414 	 * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/
8415 	 * cpu_release; those stay zero from kzalloc.
8416 	 */
8417 	if (cmd->is_cid_type) {
8418 		sch->ops_cid = *cmd->ops_cid;
8419 		sch->is_cid_type = true;
8420 	} else {
8421 		sch->ops = *cmd->ops;
8422 	}
8423 
8424 #ifdef CONFIG_EXT_SUB_SCHED
8425 	char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
8426 	if (!buf) {
8427 		ret = -ENOMEM;
8428 		goto err_free_lb_resched;
8429 	}
8430 	cgroup_path(cgrp, buf, PATH_MAX);
8431 	sch->cgrp_path = kstrdup(buf, GFP_KERNEL);
8432 	kfree(buf);
8433 	if (!sch->cgrp_path) {
8434 		ret = -ENOMEM;
8435 		goto err_free_lb_resched;
8436 	}
8437 
8438 	sch->cgrp = cgrp;
8439 	INIT_LIST_HEAD(&sch->children);
8440 	INIT_LIST_HEAD(&sch->sibling);
8441 #endif	/* CONFIG_EXT_SUB_SCHED */
8442 
8443 	/*
8444 	 * Publishing makes @sch visible to scx_prog_sched() readers. Failure
8445 	 * paths after this point must free @sch through kobject_put() whose
8446 	 * release path defers the actual freeing by an RCU grace period.
8447 	 */
8448 	rcu_assign_pointer(ops->priv, sch);
8449 
8450 	sch->kobj.kset = scx_kset;
8451 	INIT_LIST_HEAD(&sch->all);
8452 
8453 #ifdef CONFIG_EXT_SUB_SCHED
8454 	if (parent) {
8455 		/*
8456 		 * Pin @parent for @sch's lifetime. The kobject hierarchy pins
8457 		 * it only via @parent->sub_kset, which is dropped during
8458 		 * disable. Released in scx_sched_free_rcu_work().
8459 		 */
8460 		kobject_get(&parent->kobj);
8461 	}
8462 #endif	/* CONFIG_EXT_SUB_SCHED */
8463 
8464 	/*
8465 	 * Init the kobj but don't add to sysfs yet. The enable path calls
8466 	 * scx_sched_sysfs_add() once @sch's sysfs-visible state is initialized.
8467 	 */
8468 	kobject_init(&sch->kobj, &scx_ktype);
8469 
8470 	/*
8471 	 * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so
8472 	 * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid
8473 	 * drops the ref. After this point, sch owns the ref and any cleanup
8474 	 * runs through scx_sched_free_rcu_work() which puts it.
8475 	 */
8476 	sch->arena_map = cmd->arena_map;
8477 	/* BPF arena is only available on MMU && 64BIT */
8478 #if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
8479 	if (sch->arena_map)
8480 		sch->arena_kern_base = bpf_arena_map_kern_vm_start(sch->arena_map);
8481 #endif
8482 	cmd->arena_map = NULL;
8483 	return sch;
8484 
8485 #ifdef CONFIG_EXT_SUB_SCHED
8486 err_free_lb_resched:
8487 	free_cpumask_var(sch->stall_cpus);
8488 #endif
8489 err_free_lb_resched_cpumask:
8490 	free_cpumask_var(sch->bypass_lb_resched_cpumask);
8491 err_free_lb_cpumask:
8492 	free_cpumask_var(sch->bypass_lb_donee_cpumask);
8493 err_stop_helper:
8494 	kthread_destroy_worker(sch->helper);
8495 err_free_pcpu:
8496 	for_each_possible_cpu(cpu) {
8497 		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
8498 
8499 		free_cpumask_var(pcpu->cpus_to_kick);
8500 		free_cpumask_var(pcpu->cpus_to_kick_if_idle);
8501 		free_cpumask_var(pcpu->cpus_to_preempt);
8502 		free_cpumask_var(pcpu->cpus_to_preempt_lazy);
8503 		free_cpumask_var(pcpu->cpus_to_wait);
8504 	}
8505 	for_each_possible_cpu(cpu) {
8506 		if (cpu == bypass_fail_cpu)
8507 			break;
8508 		exit_dsq(scx_bypass_dsq(sch, cpu));
8509 	}
8510 	free_percpu(sch->pcpu);
8511 err_free_pnode:
8512 	for_each_node_state(node, N_POSSIBLE)
8513 		free_pnode(sch->pnode[node]);
8514 	kfree(sch->pnode);
8515 err_free_hash:
8516 	rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
8517 err_free_ei:
8518 	free_exit_info(sch->exit_info);
8519 err_free_sch:
8520 	kfree(sch);
8521 err_put_cgrp:
8522 #ifdef CONFIG_EXT_SUB_SCHED
8523 	cgroup_put(cgrp);
8524 #endif
8525 	return ERR_PTR(ret);
8526 }
8527 
8528 /*
8529  * Add @sch's kobject to sysfs, and create its sub_kset if the scheduler
8530  * implements ops.sub_attach. Called by the enable workfns once @sch's
8531  * sysfs-visible state is initialized.
8532  */
8533 int scx_sched_sysfs_add(struct scx_sched *sch)
8534 {
8535 #ifdef CONFIG_EXT_SUB_SCHED
8536 	struct scx_sched *parent = scx_parent(sch);
8537 	int ret;
8538 
8539 	if (parent)
8540 		ret = kobject_add(&sch->kobj, &parent->sub_kset->kobj,
8541 				  "sub-%llu", cgroup_id(sch_cgroup(sch)));
8542 	else
8543 		ret = kobject_add(&sch->kobj, NULL, "root");
8544 	if (ret < 0)
8545 		return ret;
8546 
8547 	if (sch->ops.sub_attach) {
8548 		sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj);
8549 		if (!sch->sub_kset)
8550 			return -ENOMEM;
8551 	}
8552 	return 0;
8553 #else
8554 	return kobject_add(&sch->kobj, NULL, "root");
8555 #endif
8556 }
8557 
8558 static int check_hotplug_seq(struct scx_sched *sch,
8559 			      const struct sched_ext_ops *ops)
8560 {
8561 	unsigned long long global_hotplug_seq;
8562 
8563 	/*
8564 	 * If a hotplug event has occurred between when a scheduler was
8565 	 * initialized, and when we were able to attach, exit and notify user
8566 	 * space about it.
8567 	 */
8568 	if (ops->hotplug_seq) {
8569 		global_hotplug_seq = atomic_long_read(&scx_hotplug_seq);
8570 		if (ops->hotplug_seq != global_hotplug_seq) {
8571 			scx_exit(sch, SCX_EXIT_UNREG_KERN,
8572 				 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
8573 				 "expected hotplug seq %llu did not match actual %llu",
8574 				 ops->hotplug_seq, global_hotplug_seq);
8575 			return -EBUSY;
8576 		}
8577 	}
8578 
8579 	return 0;
8580 }
8581 
8582 int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
8583 {
8584 	/*
8585 	 * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the
8586 	 * ops.enqueue() callback isn't implemented.
8587 	 */
8588 	if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) {
8589 		scx_error(sch, "SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented");
8590 		return -EINVAL;
8591 	}
8592 
8593 	if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) {
8594 		scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented");
8595 		return -EINVAL;
8596 	}
8597 
8598 	/*
8599 	 * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched
8600 	 * may set it to declare a dependency; reject if the root hasn't
8601 	 * enabled it.
8602 	 */
8603 	if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) &&
8604 	    !(sch->ancestors[0]->ops.flags & SCX_OPS_TID_TO_TASK)) {
8605 		scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it");
8606 		return -EINVAL;
8607 	}
8608 
8609 	/*
8610 	 * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle
8611 	 * selection policy to be enabled.
8612 	 */
8613 	if ((ops->flags & SCX_OPS_BUILTIN_IDLE_PER_NODE) &&
8614 	    (ops->update_idle && !(ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE))) {
8615 		scx_error(sch, "SCX_OPS_BUILTIN_IDLE_PER_NODE requires CPU idle selection enabled");
8616 		return -EINVAL;
8617 	}
8618 
8619 	/*
8620 	 * cid-form's struct is shorter and doesn't include the cpu_acquire /
8621 	 * cpu_release tail; reading those fields off a cid-form @ops would
8622 	 * run past the BPF allocation. Skip for cid-form.
8623 	 */
8624 	if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release))
8625 		pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
8626 
8627 	/*
8628 	 * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched
8629 	 * attaches through a cid-form-only interface (sub_attach/sub_detach),
8630 	 * and a root that accepts sub-scheds must expose cid-form state to
8631 	 * them. Reject cpu-form schedulers on either side.
8632 	 */
8633 	if (!sch->is_cid_type) {
8634 		if (scx_parent(sch)) {
8635 			scx_error(sch, "sub-sched requires cid-form struct_ops");
8636 			return -EINVAL;
8637 		}
8638 		if (ops->sub_attach || ops->sub_detach) {
8639 			scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops");
8640 			return -EINVAL;
8641 		}
8642 	}
8643 
8644 	return 0;
8645 }
8646 
8647 static void scx_root_enable_workfn(struct kthread_work *work)
8648 {
8649 	struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work);
8650 	struct sched_ext_ops *ops = cmd->ops;
8651 	struct cgroup *cgrp = root_cgroup();
8652 	struct scx_sched *sch;
8653 	struct scx_task_iter sti;
8654 	struct task_struct *p;
8655 	int i, cpu, ret;
8656 
8657 	mutex_lock(&scx_enable_mutex);
8658 
8659 	if (scx_enable_state() != SCX_DISABLED) {
8660 		ret = -EBUSY;
8661 		goto err_unlock;
8662 	}
8663 
8664 	/*
8665 	 * @ops->priv binds @ops to its scx_sched instance. It is set here by
8666 	 * scx_alloc_and_add_sched() and cleared at the tail of bpf_scx_unreg(),
8667 	 * which runs after scx_root_disable() has dropped scx_enable_mutex. If
8668 	 * it's still non-NULL here, a previous attachment on @ops has not
8669 	 * finished tearing down; proceeding would let the in-flight unreg's
8670 	 * RCU_INIT_POINTER(NULL) clobber the @ops->priv we are about to assign.
8671 	 */
8672 	if (rcu_access_pointer(ops->priv)) {
8673 		ret = -EBUSY;
8674 		goto err_unlock;
8675 	}
8676 
8677 	ret = alloc_kick_syncs();
8678 	if (ret)
8679 		goto err_unlock;
8680 
8681 	if (ops->flags & SCX_OPS_TID_TO_TASK) {
8682 		ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params);
8683 		if (ret)
8684 			goto err_free_ksyncs;
8685 	}
8686 
8687 #ifdef CONFIG_EXT_SUB_SCHED
8688 	cgroup_get(cgrp);
8689 #endif
8690 	/*
8691 	 * Transition to ENABLING to arm the disable path. Allocation failure
8692 	 * still unwinds locally. Full disabling on failure applies only after
8693 	 * scx_alloc_and_add_sched() succeeds.
8694 	 */
8695 	WARN_ON_ONCE(scx_set_enable_state(SCX_ENABLING) != SCX_DISABLED);
8696 	WARN_ON_ONCE(scx_root);
8697 
8698 	sch = scx_alloc_and_add_sched(cmd, cgrp, NULL);
8699 	if (IS_ERR(sch)) {
8700 		ret = PTR_ERR(sch);
8701 		WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_ENABLING);
8702 		goto err_free_tid_hash;
8703 	}
8704 
8705 	if (sch->is_cid_type)
8706 		static_branch_enable(&__scx_is_cid_type);
8707 
8708 	atomic_long_set(&scx_nr_rejected, 0);
8709 
8710 	for_each_possible_cpu(cpu) {
8711 		struct rq *rq = cpu_rq(cpu);
8712 
8713 		rq->scx.local_dsq.sched = sch;
8714 		rq->scx.cpuperf_target = SCX_CPUPERF_ONE;
8715 	}
8716 
8717 	scx_discard_stale_ecaps_syncs();
8718 	scx_rescue_set_knobs(sch);
8719 
8720 	/*
8721 	 * Keep CPUs stable during enable so that the BPF scheduler can track
8722 	 * online CPUs by watching ->on/offline_cpu() after ->init().
8723 	 */
8724 	cpus_read_lock();
8725 
8726 	/*
8727 	 * Build the cid mapping into a private under-construction set. It
8728 	 * becomes visible to readers only through scx_cid_publish_tables() once
8729 	 * ops.init_cids() has finalized the layout.
8730 	 */
8731 	ret = scx_cid_init(sch);
8732 	if (ret) {
8733 		cpus_read_unlock();
8734 		goto err_disable;
8735 	}
8736 
8737 	scx_init_rq_ops_online();
8738 
8739 	/*
8740 	 * Make the scheduler instance visible. Must be inside cpus_read_lock().
8741 	 * See handle_hotplug().
8742 	 */
8743 	rcu_assign_pointer(scx_root, sch);
8744 
8745 	ret = scx_link_sched(sch);
8746 	if (ret) {
8747 		cpus_read_unlock();
8748 		goto err_disable;
8749 	}
8750 
8751 	scx_idle_enable(ops);
8752 
8753 	/*
8754 	 * A cid-form scheduler finalizes its cid layout in ops.init_cids(),
8755 	 * which may call scx_bpf_cid_override(). Run it before the caps and
8756 	 * shard state are built so the final layout is in effect.
8757 	 */
8758 	if (sch->is_cid_type && sch->ops_cid.init_cids) {
8759 		ret = SCX_CALL_OP_RET(sch, init_cids, NULL);
8760 		if (ret) {
8761 			ret = scx_ops_sanitize_err(sch, "init_cids", ret);
8762 			cpus_read_unlock();
8763 			scx_error(sch, "ops.init_cids() failed (%d)", ret);
8764 			goto err_disable;
8765 		}
8766 	}
8767 
8768 	/* the cid layout is final, expose it to readers */
8769 	scx_cid_publish_tables();
8770 
8771 	ret = scx_arena_pool_init(sch);
8772 	if (ret) {
8773 		cpus_read_unlock();
8774 		goto err_disable;
8775 	}
8776 
8777 	ret = scx_alloc_kern_arena_objs(sch);
8778 	if (ret) {
8779 		cpus_read_unlock();
8780 		goto err_disable;
8781 	}
8782 
8783 	ret = scx_alloc_pshards(sch);
8784 	if (ret) {
8785 		cpus_read_unlock();
8786 		goto err_disable;
8787 	}
8788 
8789 	scx_init_root_caps(sch);
8790 
8791 	/* the cid caps and shards are live now, so ops.init() can query them */
8792 	if (sch->ops.init) {
8793 		ret = SCX_CALL_OP_RET(sch, init, NULL);
8794 		if (ret) {
8795 			ret = scx_ops_sanitize_err(sch, "init", ret);
8796 			cpus_read_unlock();
8797 			scx_error(sch, "ops.init() failed (%d)", ret);
8798 			goto err_disable;
8799 		}
8800 		sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
8801 	}
8802 
8803 	ret = scx_sched_sysfs_add(sch);
8804 	if (ret) {
8805 		cpus_read_unlock();
8806 		goto err_disable;
8807 	}
8808 
8809 	for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)
8810 		if (((void (**)(void))ops)[i])
8811 			set_bit(i, sch->has_op);
8812 
8813 	ret = check_hotplug_seq(sch, ops);
8814 	if (ret) {
8815 		cpus_read_unlock();
8816 		goto err_disable;
8817 	}
8818 	scx_idle_update_selcpu_topology(ops);
8819 
8820 	cpus_read_unlock();
8821 
8822 	ret = scx_validate_ops(sch, ops);
8823 	if (ret)
8824 		goto err_disable;
8825 
8826 	/*
8827 	 * Attach the ext_server bandwidth reservation before anything is
8828 	 * committed so that we can fail the enable if the root domain cannot
8829 	 * accommodate it. The matching fair_server detach is deferred to the
8830 	 * tail of this function, after the switch is fully committed and can no
8831 	 * longer fail.
8832 	 *
8833 	 * On failure, err_disable funnels into scx_root_disable() which
8834 	 * detaches ext_server, so partially-attached state is cleaned up
8835 	 * automatically.
8836 	 */
8837 	for_each_possible_cpu(cpu) {
8838 		struct rq *rq = cpu_rq(cpu);
8839 
8840 		scoped_guard(rq_lock_irqsave, rq) {
8841 			update_rq_clock(rq);
8842 			ret = dl_server_attach_bw(&rq->ext_server);
8843 		}
8844 		if (ret) {
8845 			pr_warn("sched_ext: failed to attach ext_server on CPU %d (%d)\n",
8846 				cpu, ret);
8847 			goto err_disable;
8848 		}
8849 	}
8850 
8851 	/*
8852 	 * Once __scx_enabled is set, %current can be switched to SCX anytime.
8853 	 * This can lead to stalls as some BPF schedulers (e.g. userspace
8854 	 * scheduling) may not function correctly before all tasks are switched.
8855 	 * Init in bypass mode to guarantee forward progress.
8856 	 */
8857 	scx_bypass(sch, true);
8858 
8859 	for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++)
8860 		if (((void (**)(void))ops)[i])
8861 			set_bit(i, sch->has_op);
8862 
8863 	scx_ops_cid_sched_updated_enable(sch);
8864 
8865 	if (sch->ops.cpu_acquire || sch->ops.cpu_release)
8866 		sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT;
8867 
8868 	/*
8869 	 * Lock out forks, cgroup on/offlining and moves before opening the
8870 	 * floodgate so that they don't wander into the operations prematurely.
8871 	 */
8872 	percpu_down_write(&scx_fork_rwsem);
8873 
8874 	WARN_ON_ONCE(scx_init_task_enabled);
8875 	scx_init_task_enabled = true;
8876 
8877 	/* flip under fork_rwsem; the iter below covers existing tasks */
8878 	if (ops->flags & SCX_OPS_TID_TO_TASK)
8879 		static_branch_enable(&__scx_tid_to_task_enabled);
8880 
8881 	/*
8882 	 * Enable ops for every task. Fork is excluded by scx_fork_rwsem
8883 	 * preventing new tasks from being added. No need to exclude tasks
8884 	 * leaving as sched_ext_dead() can handle both prepped and enabled
8885 	 * tasks. Prep all tasks first and then enable them with preemption
8886 	 * disabled.
8887 	 *
8888 	 * All cgroups should be initialized before scx_init_task() so that the
8889 	 * BPF scheduler can reliably track each task's cgroup membership from
8890 	 * scx_init_task(). Lock out cgroup on/offlining and task migrations
8891 	 * while tasks are being initialized so that scx_cgroup_can_attach()
8892 	 * never sees uninitialized tasks.
8893 	 */
8894 	scx_cgroup_lock();
8895 	set_cgroup_sched(sch_cgroup(sch), sch);
8896 	ret = scx_cgroup_init(sch);
8897 	if (ret)
8898 		goto err_disable_unlock_all;
8899 
8900 	WARN_ON_ONCE(scx_cgroup_enabled);
8901 	scx_cgroup_enabled = true;
8902 
8903 	scx_task_iter_start(&sti, NULL);
8904 	while ((p = scx_task_iter_next_locked(&sti))) {
8905 		/*
8906 		 * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD
8907 		 * tasks are filtered by scx_task_iter_next_locked().
8908 		 * sched_ext_dead() removes @p from scx_tasks under the same
8909 		 * lock before put_task_struct_rcu_user() runs, so @p->usage
8910 		 * is guaranteed > 0 here.
8911 		 */
8912 		get_task_struct(p);
8913 
8914 		/*
8915 		 * Set %INIT_BEGIN under the iter's rq lock so that a concurrent
8916 		 * sched_ext_dead() does not call ops.exit_task() on @p while
8917 		 * ops.init_task() is running. If sched_ext_dead() runs before
8918 		 * this store, it has already removed @p from scx_tasks and the
8919 		 * iter won't visit @p; if it runs after, it observes
8920 		 * %INIT_BEGIN and transitions to %DEAD without calling ops,
8921 		 * leaving the post-init recheck below to unwind.
8922 		 */
8923 		scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
8924 		scx_task_iter_unlock(&sti);
8925 
8926 		ret = __scx_init_task(sch, p, NULL, false);
8927 
8928 		scx_task_iter_relock(&sti, p);
8929 
8930 		if (unlikely(ret)) {
8931 			if (scx_get_task_state(p) != SCX_TASK_DEAD)
8932 				scx_set_task_state(p, SCX_TASK_NONE);
8933 			scx_task_iter_stop(&sti);
8934 			scx_error(sch, "ops.init_task() failed (%d) for %s[%d]",
8935 				  ret, p->comm, p->pid);
8936 			put_task_struct(p);
8937 			goto err_disable_unlock_all;
8938 		}
8939 
8940 		if (scx_get_task_state(p) == SCX_TASK_DEAD) {
8941 			/*
8942 			 * sched_ext_dead() observed %INIT_BEGIN and set %DEAD.
8943 			 * ops.exit_task() is owed to the sched __scx_init_task()
8944 			 * ran against; call it now.
8945 			 */
8946 			scx_sub_init_cancel_task(sch, p);
8947 		} else {
8948 			scx_set_task_state(p, SCX_TASK_INIT);
8949 			scx_set_task_sched(p, sch);
8950 			scx_set_task_state(p, SCX_TASK_READY);
8951 		}
8952 
8953 		/*
8954 		 * Insert into the tid hash. scx_tasks_lock is held by the iter;
8955 		 * list_empty() guards against sched_ext_dead() having taken @p
8956 		 * off the list while init ran unlocked.
8957 		 */
8958 		if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node))
8959 			scx_tid_hash_insert(p);
8960 
8961 		put_task_struct(p);
8962 	}
8963 	scx_task_iter_stop(&sti);
8964 	scx_cgroup_unlock();
8965 	percpu_up_write(&scx_fork_rwsem);
8966 
8967 	/*
8968 	 * All tasks are READY. It's safe to turn on scx_enabled() and switch
8969 	 * all eligible tasks.
8970 	 */
8971 	WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL));
8972 	static_branch_enable(&__scx_enabled);
8973 
8974 	/*
8975 	 * We're fully committed and can't fail. The task READY -> ENABLED
8976 	 * transitions here are synchronized against sched_ext_dead() through
8977 	 * scx_tasks_lock.
8978 	 */
8979 	percpu_down_write(&scx_fork_rwsem);
8980 	scx_task_iter_start(&sti, NULL);
8981 	while ((p = scx_task_iter_next_locked(&sti))) {
8982 		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
8983 		const struct sched_class *old_class = p->sched_class;
8984 		const struct sched_class *new_class = scx_setscheduler_class(p);
8985 
8986 		if (scx_get_task_state(p) != SCX_TASK_READY)
8987 			continue;
8988 
8989 		if (old_class != new_class)
8990 			queue_flags |= DEQUEUE_CLASS;
8991 		if (new_class == &ext_sched_class) {
8992 			scx_prepare_task_sched_change(p);
8993 			queue_flags |= DEQUEUE_NOCLOCK;
8994 		}
8995 
8996 		scoped_guard (sched_change, p, queue_flags) {
8997 			scx_set_task_slice(p, READ_ONCE(sch->slice_dfl));
8998 			p->sched_class = new_class;
8999 		}
9000 	}
9001 	scx_task_iter_stop(&sti);
9002 	percpu_up_write(&scx_fork_rwsem);
9003 
9004 	scx_bypass(sch, false);
9005 
9006 	if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) {
9007 		WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE);
9008 		ret = -EBUSY;
9009 		goto err_disable;
9010 	}
9011 
9012 	if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
9013 		static_branch_enable(&__scx_switched_all);
9014 
9015 	/*
9016 	 * Detach the fair_server bandwidth reservation now that the switch
9017 	 * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no
9018 	 * task will ever run in the fair class, so give that bandwidth
9019 	 * back to the RT class. The matching ext_server attach already
9020 	 * happened earlier; this only releases bandwidth and cannot fail.
9021 	 *
9022 	 * In partial mode keep fair_server attached.
9023 	 */
9024 	if (scx_switched_all()) {
9025 		for_each_possible_cpu(cpu) {
9026 			struct rq *rq = cpu_rq(cpu);
9027 
9028 			guard(rq_lock_irqsave)(rq);
9029 			update_rq_clock(rq);
9030 			dl_server_detach_bw(&rq->fair_server);
9031 		}
9032 	}
9033 
9034 	pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n",
9035 		sch->ops.name, scx_switched_all() ? "" : " (partial)");
9036 	kobject_uevent(&sch->kobj, KOBJ_ADD);
9037 	mutex_unlock(&scx_enable_mutex);
9038 
9039 	atomic_long_inc(&scx_enable_seq);
9040 
9041 	cmd->ret = 0;
9042 	return;
9043 
9044 err_free_tid_hash:
9045 	if (ops->flags & SCX_OPS_TID_TO_TASK)
9046 		rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
9047 err_free_ksyncs:
9048 	free_kick_syncs();
9049 err_unlock:
9050 	mutex_unlock(&scx_enable_mutex);
9051 	cmd->ret = ret;
9052 	return;
9053 
9054 err_disable_unlock_all:
9055 	scx_cgroup_unlock();
9056 	percpu_up_write(&scx_fork_rwsem);
9057 	/* we'll soon enter disable path, keep bypass on */
9058 err_disable:
9059 	mutex_unlock(&scx_enable_mutex);
9060 	/*
9061 	 * Returning an error code here would not pass all the error information
9062 	 * to userspace. Record errno using scx_error() for cases scx_error()
9063 	 * wasn't already invoked and exit indicating success so that the error
9064 	 * is notified through ops.exit() with all the details.
9065 	 *
9066 	 * Flush scx_disable_work to ensure that error is reported before init
9067 	 * completion. sch's base reference will be put by bpf_scx_unreg().
9068 	 */
9069 	scx_error(sch, "scx_root_enable() failed (%d)", ret);
9070 	scx_flush_disable_work(sch);
9071 	cmd->ret = 0;
9072 }
9073 
9074 static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link)
9075 {
9076 	static struct kthread_worker *helper;
9077 	static DEFINE_MUTEX(helper_mutex);
9078 
9079 	if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) {
9080 		pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n");
9081 		return -EINVAL;
9082 	}
9083 
9084 	if (!READ_ONCE(helper)) {
9085 		mutex_lock(&helper_mutex);
9086 		if (!helper) {
9087 			struct kthread_worker *w =
9088 				kthread_run_worker(0, "scx_enable_helper");
9089 			if (IS_ERR_OR_NULL(w)) {
9090 				mutex_unlock(&helper_mutex);
9091 				return -ENOMEM;
9092 			}
9093 			sched_set_fifo(w->task);
9094 			WRITE_ONCE(helper, w);
9095 		}
9096 		mutex_unlock(&helper_mutex);
9097 	}
9098 
9099 #ifdef CONFIG_EXT_SUB_SCHED
9100 	if (cmd->ops->sub_cgroup_id > 1)
9101 		kthread_init_work(&cmd->work, scx_sub_enable_workfn);
9102 	else
9103 #endif	/* CONFIG_EXT_SUB_SCHED */
9104 		kthread_init_work(&cmd->work, scx_root_enable_workfn);
9105 
9106 	kthread_queue_work(READ_ONCE(helper), &cmd->work);
9107 	kthread_flush_work(&cmd->work);
9108 	return cmd->ret;
9109 }
9110 
9111 
9112 /********************************************************************************
9113  * bpf_struct_ops plumbing.
9114  */
9115 #include <linux/bpf_verifier.h>
9116 #include <linux/bpf.h>
9117 #include <linux/btf.h>
9118 
9119 static const struct btf_type *task_struct_type;
9120 
9121 static bool bpf_scx_is_valid_access(int off, int size,
9122 				    enum bpf_access_type type,
9123 				    const struct bpf_prog *prog,
9124 				    struct bpf_insn_access_aux *info)
9125 {
9126 	if (type != BPF_READ)
9127 		return false;
9128 	if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
9129 		return false;
9130 	if (off % size != 0)
9131 		return false;
9132 
9133 	return btf_ctx_access(off, size, type, prog, info);
9134 }
9135 
9136 /* common to both forms: only scx.disallow is writable */
9137 static int bpf_scx_btf_struct_access_common(const struct bpf_reg_state *reg,
9138 					    int off, int size)
9139 {
9140 	const struct btf_type *t;
9141 
9142 	t = btf_type_by_id(reg->btf, reg->btf_id);
9143 	if (t == task_struct_type &&
9144 	    off >= offsetof(struct task_struct, scx.disallow) &&
9145 	    off + size <= offsetofend(struct task_struct, scx.disallow))
9146 		return SCALAR_VALUE;
9147 
9148 	return -EACCES;
9149 }
9150 
9151 static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
9152 				     const struct bpf_reg_state *reg, int off,
9153 				     int size)
9154 {
9155 	const struct btf_type *t;
9156 
9157 	t = btf_type_by_id(reg->btf, reg->btf_id);
9158 	if (t == task_struct_type) {
9159 		if ((off >= offsetof(struct task_struct, scx.slice) &&
9160 		     off + size <= offsetofend(struct task_struct, scx.slice)) ||
9161 		    (off >= offsetof(struct task_struct, scx.dsq_vtime) &&
9162 		     off + size <= offsetofend(struct task_struct, scx.dsq_vtime)))
9163 			return SCALAR_VALUE;
9164 	}
9165 
9166 	return bpf_scx_btf_struct_access_common(reg, off, size);
9167 }
9168 
9169 /* cid-form rejects direct slice and dsq_vtime writes in favor of the kfuncs */
9170 static int bpf_scx_cid_btf_struct_access(struct bpf_verifier_log *log,
9171 					 const struct bpf_reg_state *reg, int off,
9172 					 int size)
9173 {
9174 	return bpf_scx_btf_struct_access_common(reg, off, size);
9175 }
9176 
9177 static const struct bpf_verifier_ops bpf_scx_verifier_ops = {
9178 	.get_func_proto = bpf_base_func_proto,
9179 	.is_valid_access = bpf_scx_is_valid_access,
9180 	.btf_struct_access = bpf_scx_btf_struct_access,
9181 };
9182 
9183 static const struct bpf_verifier_ops bpf_scx_cid_verifier_ops = {
9184 	.get_func_proto = bpf_base_func_proto,
9185 	.is_valid_access = bpf_scx_is_valid_access,
9186 	.btf_struct_access = bpf_scx_cid_btf_struct_access,
9187 };
9188 
9189 static int bpf_scx_init_member(const struct btf_type *t,
9190 			       const struct btf_member *member,
9191 			       void *kdata, const void *udata)
9192 {
9193 	const struct sched_ext_ops *uops = udata;
9194 	struct sched_ext_ops *ops = kdata;
9195 	u32 moff = __btf_member_bit_offset(t, member) / 8;
9196 	int ret;
9197 
9198 	switch (moff) {
9199 	case offsetof(struct sched_ext_ops, dispatch_max_batch):
9200 		if (*(u32 *)(udata + moff) > INT_MAX)
9201 			return -E2BIG;
9202 		ops->dispatch_max_batch = *(u32 *)(udata + moff);
9203 		return 1;
9204 	case offsetof(struct sched_ext_ops, flags):
9205 		if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS)
9206 			return -EINVAL;
9207 		ops->flags = *(u64 *)(udata + moff);
9208 		return 1;
9209 	case offsetof(struct sched_ext_ops, name):
9210 		ret = bpf_obj_name_cpy(ops->name, uops->name,
9211 				       sizeof(ops->name));
9212 		if (ret < 0)
9213 			return ret;
9214 		if (ret == 0)
9215 			return -EINVAL;
9216 		return 1;
9217 	case offsetof(struct sched_ext_ops, timeout_ms):
9218 		if (msecs_to_jiffies(*(u32 *)(udata + moff)) >
9219 		    SCX_WATCHDOG_MAX_TIMEOUT)
9220 			return -E2BIG;
9221 		ops->timeout_ms = *(u32 *)(udata + moff);
9222 		return 1;
9223 	case offsetof(struct sched_ext_ops, exit_dump_len):
9224 		ops->exit_dump_len =
9225 			*(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN;
9226 		return 1;
9227 	case offsetof(struct sched_ext_ops, hotplug_seq):
9228 		ops->hotplug_seq = *(u64 *)(udata + moff);
9229 		return 1;
9230 	case offsetof(struct sched_ext_ops, cid_shard_size):
9231 		ops->cid_shard_size = *(u32 *)(udata + moff);
9232 		return 1;
9233 	case offsetof(struct sched_ext_ops, rescue_bandwidth_ppt): {
9234 		u32 bw_ppt = *(u32 *)(udata + moff);
9235 
9236 		if (bw_ppt > SCX_RESCUE_MAX_BW_PPT && bw_ppt != SCX_RESCUE_DISABLE)
9237 			return -E2BIG;
9238 		ops->rescue_bandwidth_ppt = bw_ppt;
9239 		return 1;
9240 	}
9241 	case offsetof(struct sched_ext_ops, rescue_quantum_us): {
9242 		u32 quantum_us = *(u32 *)(udata + moff);
9243 
9244 		if (quantum_us > SCX_RESCUE_MAX_QUANTUM_US)
9245 			return -E2BIG;
9246 		if (quantum_us && quantum_us < SCX_RESCUE_MIN_QUANTUM_US)
9247 			return -EINVAL;
9248 		ops->rescue_quantum_us = quantum_us;
9249 		return 1;
9250 	}
9251 #ifdef CONFIG_EXT_SUB_SCHED
9252 	case offsetof(struct sched_ext_ops, sub_cgroup_id):
9253 		ops->sub_cgroup_id = *(u64 *)(udata + moff);
9254 		return 1;
9255 #endif	/* CONFIG_EXT_SUB_SCHED */
9256 	}
9257 
9258 	return 0;
9259 }
9260 
9261 static int bpf_scx_check_member(const struct btf_type *t,
9262 				const struct btf_member *member,
9263 				const struct bpf_prog *prog)
9264 {
9265 	u32 moff = __btf_member_bit_offset(t, member) / 8;
9266 
9267 	switch (moff) {
9268 	case offsetof(struct sched_ext_ops, init_task):
9269 #ifdef CONFIG_EXT_GROUP_SCHED
9270 	case offsetof(struct sched_ext_ops, cgroup_init):
9271 	case offsetof(struct sched_ext_ops, cgroup_exit):
9272 	case offsetof(struct sched_ext_ops, cgroup_prep_move):
9273 	case offsetof(struct sched_ext_ops, cgroup_set_bandwidth):
9274 #endif
9275 	case offsetof(struct sched_ext_ops, cpu_online):
9276 	case offsetof(struct sched_ext_ops, cpu_offline):
9277 	case offsetof(struct sched_ext_ops, init_cids):
9278 	case offsetof(struct sched_ext_ops, init):
9279 	case offsetof(struct sched_ext_ops, exit):
9280 	case offsetof(struct sched_ext_ops, sub_attach):
9281 	case offsetof(struct sched_ext_ops, sub_detach):
9282 		break;
9283 	default:
9284 		if (prog->sleepable)
9285 			return -EINVAL;
9286 	}
9287 
9288 #ifdef CONFIG_EXT_SUB_SCHED
9289 	/*
9290 	 * Enable private stack for operations that can nest along the
9291 	 * hierarchy.
9292 	 *
9293 	 * XXX - Ideally, we should only do this for scheds that allow
9294 	 * sub-scheds and sub-scheds themselves but I don't know how to access
9295 	 * struct_ops from here.
9296 	 */
9297 	switch (moff) {
9298 	case offsetof(struct sched_ext_ops, dispatch):
9299 		prog->aux->priv_stack_requested = true;
9300 		prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch;
9301 		break;
9302 	case offsetof(struct sched_ext_ops, sub_caps_updated):
9303 		prog->aux->priv_stack_requested = true;
9304 		prog->aux->recursion_detected = scx_pstack_recursion_on_caps_updated;
9305 		break;
9306 	}
9307 #endif	/* CONFIG_EXT_SUB_SCHED */
9308 
9309 	return 0;
9310 }
9311 
9312 static int bpf_scx_reg(void *kdata, struct bpf_link *link)
9313 {
9314 	struct scx_enable_cmd cmd = { .ops = kdata };
9315 
9316 	return scx_enable(&cmd, link);
9317 }
9318 
9319 struct scx_arena_scan {
9320 	struct bpf_map	*arena;
9321 	int		err;
9322 };
9323 
9324 /*
9325  * The verifier enforces one arena per BPF program, so each struct_ops
9326  * member prog contributes at most one arena via bpf_prog_arena().
9327  * Require all non-NULL contributions to match.
9328  */
9329 static int scx_arena_scan_prog(struct bpf_prog *prog, void *data)
9330 {
9331 	struct scx_arena_scan *s = data;
9332 	struct bpf_map *arena = NULL;
9333 
9334 	/* arena.o, which defines these, is built only on MMU && 64BIT */
9335 #if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
9336 	arena = bpf_prog_arena(prog);
9337 #endif
9338 	if (!arena)
9339 		return 0;
9340 	if (s->arena && s->arena != arena) {
9341 		s->err = -EINVAL;
9342 		return 1;
9343 	}
9344 	s->arena = arena;
9345 	return 0;
9346 }
9347 
9348 static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link)
9349 {
9350 	struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true };
9351 	struct scx_arena_scan scan = {};
9352 	int ret;
9353 
9354 	bpf_struct_ops_for_each_prog(kdata, scx_arena_scan_prog, &scan);
9355 	if (scan.err) {
9356 		pr_err("sched_ext: cid-form scheduler uses multiple arena maps\n");
9357 		return scan.err;
9358 	}
9359 	if (!scan.arena) {
9360 		pr_err("sched_ext: cid-form scheduler must use a BPF arena map\n");
9361 		return -EINVAL;
9362 	}
9363 
9364 	bpf_map_inc(scan.arena);
9365 	cmd.arena_map = scan.arena;
9366 	ret = scx_enable(&cmd, link);
9367 	if (cmd.arena_map)		/* not consumed by scx_alloc_and_add_sched() */
9368 		bpf_map_put(cmd.arena_map);
9369 	return ret;
9370 }
9371 
9372 static void bpf_scx_unreg(void *kdata, struct bpf_link *link)
9373 {
9374 	struct sched_ext_ops *ops = kdata;
9375 	struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);
9376 
9377 	scx_disable(sch, SCX_EXIT_UNREG);
9378 	scx_flush_disable_work(sch);
9379 	RCU_INIT_POINTER(ops->priv, NULL);
9380 	kobject_put(&sch->kobj);
9381 }
9382 
9383 static int bpf_scx_init(struct btf *btf)
9384 {
9385 	task_struct_type = btf_type_by_id(btf, btf_tracing_ids[BTF_TRACING_TYPE_TASK]);
9386 
9387 	return 0;
9388 }
9389 
9390 static int bpf_scx_update(void *kdata, void *old_kdata, struct bpf_link *link)
9391 {
9392 	/*
9393 	 * sched_ext does not support updating the actively-loaded BPF
9394 	 * scheduler, as registering a BPF scheduler can always fail if the
9395 	 * scheduler returns an error code for e.g. ops.init(), ops.init_task(),
9396 	 * etc. Similarly, we can always race with unregistration happening
9397 	 * elsewhere, such as with sysrq.
9398 	 */
9399 	return -EOPNOTSUPP;
9400 }
9401 
9402 static int bpf_scx_validate(void *kdata)
9403 {
9404 	return 0;
9405 }
9406 
9407 static s32 sched_ext_ops__select_cpu(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; }
9408 static void sched_ext_ops__enqueue(struct task_struct *p, u64 enq_flags) {}
9409 static void sched_ext_ops__dequeue(struct task_struct *p, u64 enq_flags) {}
9410 static void sched_ext_ops__dispatch(s32 prev_cpu, struct task_struct *prev__nullable) {}
9411 static void sched_ext_ops__tick(struct task_struct *p) {}
9412 static void sched_ext_ops__runnable(struct task_struct *p, u64 enq_flags) {}
9413 static void sched_ext_ops__running(struct task_struct *p) {}
9414 static void sched_ext_ops__stopping(struct task_struct *p, bool runnable) {}
9415 static void sched_ext_ops__quiescent(struct task_struct *p, u64 deq_flags) {}
9416 static bool sched_ext_ops__yield(struct task_struct *from, struct task_struct *to__nullable) { return false; }
9417 static bool sched_ext_ops__core_sched_before(struct task_struct *a, struct task_struct *b) { return false; }
9418 static void sched_ext_ops__set_weight(struct task_struct *p, u32 weight) {}
9419 static void sched_ext_ops__set_cpumask(struct task_struct *p, const struct cpumask *mask) {}
9420 static void sched_ext_ops__update_idle(s32 cpu, bool idle) {}
9421 static void sched_ext_ops__cpu_acquire(s32 cpu, struct scx_cpu_acquire_args *args) {}
9422 static void sched_ext_ops__cpu_release(s32 cpu, struct scx_cpu_release_args *args) {}
9423 static s32 sched_ext_ops__init_task(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; }
9424 static void sched_ext_ops__exit_task(struct task_struct *p, struct scx_exit_task_args *args) {}
9425 static void sched_ext_ops__enable(struct task_struct *p) {}
9426 static void sched_ext_ops__disable(struct task_struct *p) {}
9427 #ifdef CONFIG_EXT_GROUP_SCHED
9428 static s32 sched_ext_ops__cgroup_init(struct cgroup *cgrp, struct scx_cgroup_init_args *args) { return -EINVAL; }
9429 static void sched_ext_ops__cgroup_exit(struct cgroup *cgrp) {}
9430 static s32 sched_ext_ops__cgroup_prep_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) { return -EINVAL; }
9431 static void sched_ext_ops__cgroup_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {}
9432 static void sched_ext_ops__cgroup_cancel_move(struct task_struct *p, struct cgroup *from, struct cgroup *to) {}
9433 static void sched_ext_ops__cgroup_set_weight(struct cgroup *cgrp, u32 weight) {}
9434 static void sched_ext_ops__cgroup_set_bandwidth(struct cgroup *cgrp, u64 period_us, u64 quota_us, u64 burst_us) {}
9435 static void sched_ext_ops__cgroup_set_idle(struct cgroup *cgrp, bool idle) {}
9436 #endif	/* CONFIG_EXT_GROUP_SCHED */
9437 static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return -EINVAL; }
9438 static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {}
9439 static void sched_ext_ops__cpu_online(s32 cpu) {}
9440 static void sched_ext_ops__cpu_offline(s32 cpu) {}
9441 static s32 sched_ext_ops__init_cids(void) { return -EINVAL; }
9442 static s32 sched_ext_ops__init(void) { return -EINVAL; }
9443 static void sched_ext_ops__exit(struct scx_exit_info *info) {}
9444 static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {}
9445 static void sched_ext_ops__dump_cpu(struct scx_dump_ctx *ctx, s32 cpu, bool idle) {}
9446 static void sched_ext_ops__dump_task(struct scx_dump_ctx *ctx, struct task_struct *p) {}
9447 
9448 static struct sched_ext_ops __bpf_ops_sched_ext_ops = {
9449 	.select_cpu		= sched_ext_ops__select_cpu,
9450 	.enqueue		= sched_ext_ops__enqueue,
9451 	.dequeue		= sched_ext_ops__dequeue,
9452 	.dispatch		= sched_ext_ops__dispatch,
9453 	.tick			= sched_ext_ops__tick,
9454 	.runnable		= sched_ext_ops__runnable,
9455 	.running		= sched_ext_ops__running,
9456 	.stopping		= sched_ext_ops__stopping,
9457 	.quiescent		= sched_ext_ops__quiescent,
9458 	.yield			= sched_ext_ops__yield,
9459 	.core_sched_before	= sched_ext_ops__core_sched_before,
9460 	.set_weight		= sched_ext_ops__set_weight,
9461 	.set_cpumask		= sched_ext_ops__set_cpumask,
9462 	.update_idle		= sched_ext_ops__update_idle,
9463 	.cpu_acquire		= sched_ext_ops__cpu_acquire,
9464 	.cpu_release		= sched_ext_ops__cpu_release,
9465 	.init_task		= sched_ext_ops__init_task,
9466 	.exit_task		= sched_ext_ops__exit_task,
9467 	.enable			= sched_ext_ops__enable,
9468 	.disable		= sched_ext_ops__disable,
9469 #ifdef CONFIG_EXT_GROUP_SCHED
9470 	.cgroup_init		= sched_ext_ops__cgroup_init,
9471 	.cgroup_exit		= sched_ext_ops__cgroup_exit,
9472 	.cgroup_prep_move	= sched_ext_ops__cgroup_prep_move,
9473 	.cgroup_move		= sched_ext_ops__cgroup_move,
9474 	.cgroup_cancel_move	= sched_ext_ops__cgroup_cancel_move,
9475 	.cgroup_set_weight	= sched_ext_ops__cgroup_set_weight,
9476 	.cgroup_set_bandwidth	= sched_ext_ops__cgroup_set_bandwidth,
9477 	.cgroup_set_idle	= sched_ext_ops__cgroup_set_idle,
9478 #endif
9479 	.sub_attach		= sched_ext_ops__sub_attach,
9480 	.sub_detach		= sched_ext_ops__sub_detach,
9481 	.cpu_online		= sched_ext_ops__cpu_online,
9482 	.cpu_offline		= sched_ext_ops__cpu_offline,
9483 	.init_cids		= sched_ext_ops__init_cids,
9484 	.init			= sched_ext_ops__init,
9485 	.exit			= sched_ext_ops__exit,
9486 	.dump			= sched_ext_ops__dump,
9487 	.dump_cpu		= sched_ext_ops__dump_cpu,
9488 	.dump_task		= sched_ext_ops__dump_task,
9489 };
9490 
9491 static struct bpf_struct_ops bpf_sched_ext_ops = {
9492 	.verifier_ops = &bpf_scx_verifier_ops,
9493 	.reg = bpf_scx_reg,
9494 	.unreg = bpf_scx_unreg,
9495 	.check_member = bpf_scx_check_member,
9496 	.init_member = bpf_scx_init_member,
9497 	.init = bpf_scx_init,
9498 	.update = bpf_scx_update,
9499 	.validate = bpf_scx_validate,
9500 	.name = "sched_ext_ops",
9501 	.owner = THIS_MODULE,
9502 	.cfi_stubs = &__bpf_ops_sched_ext_ops
9503 };
9504 
9505 /*
9506  * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types
9507  * identical, only param names differ across structs) are reused. Some need
9508  * fresh stubs, set_cmask and enable due to argument differences and the
9509  * sub-sched notifiers because no cpu-form stub exists to reuse.
9510  */
9511 static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {}
9512 static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {}
9513 static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {}
9514 static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {}
9515 static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {}
9516 static void sched_ext_ops__sub_child_ecaps_updated(u64 cgroup_id, s32 cid, u64 before, u64 after) {}
9517 
9518 static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
9519 	.select_cid		= sched_ext_ops__select_cpu,
9520 	.enqueue		= sched_ext_ops__enqueue,
9521 	.dequeue		= sched_ext_ops__dequeue,
9522 	.dispatch		= sched_ext_ops__dispatch,
9523 	.tick			= sched_ext_ops__tick,
9524 	.runnable		= sched_ext_ops__runnable,
9525 	.running		= sched_ext_ops__running,
9526 	.stopping		= sched_ext_ops__stopping,
9527 	.quiescent		= sched_ext_ops__quiescent,
9528 	.yield			= sched_ext_ops__yield,
9529 	.core_sched_before	= sched_ext_ops__core_sched_before,
9530 	.set_weight		= sched_ext_ops__set_weight,
9531 	.set_cmask		= sched_ext_ops_cid__set_cmask,
9532 	.update_idle		= sched_ext_ops__update_idle,
9533 	.init_task		= sched_ext_ops__init_task,
9534 	.exit_task		= sched_ext_ops__exit_task,
9535 	.enable			= sched_ext_ops_cid__enable,
9536 	.disable		= sched_ext_ops__disable,
9537 #ifdef CONFIG_EXT_GROUP_SCHED
9538 	.cpuctl_init		= sched_ext_ops__cgroup_init,
9539 	.cpuctl_exit		= sched_ext_ops__cgroup_exit,
9540 	.cpuctl_prep_move	= sched_ext_ops__cgroup_prep_move,
9541 	.cpuctl_move		= sched_ext_ops__cgroup_move,
9542 	.cpuctl_cancel_move	= sched_ext_ops__cgroup_cancel_move,
9543 	.cpuctl_set_weight	= sched_ext_ops__cgroup_set_weight,
9544 	.cpuctl_set_bandwidth	= sched_ext_ops__cgroup_set_bandwidth,
9545 	.cpuctl_set_idle	= sched_ext_ops__cgroup_set_idle,
9546 #endif
9547 	.sub_attach		= sched_ext_ops__sub_attach,
9548 	.sub_detach		= sched_ext_ops__sub_detach,
9549 	.sub_caps_updated	= sched_ext_ops__sub_caps_updated,
9550 	.sub_ecaps_updated	= sched_ext_ops__sub_ecaps_updated,
9551 	.sub_cid_sched_updated	= sched_ext_ops__sub_cid_sched_updated,
9552 	.sub_child_ecaps_updated = sched_ext_ops__sub_child_ecaps_updated,
9553 	.cid_online		= sched_ext_ops__cpu_online,
9554 	.cid_offline		= sched_ext_ops__cpu_offline,
9555 	.init_cids		= sched_ext_ops__init_cids,
9556 	.init			= sched_ext_ops__init,
9557 	.exit			= sched_ext_ops__exit,
9558 	.dump			= sched_ext_ops__dump,
9559 	.dump_cid		= sched_ext_ops__dump_cpu,
9560 	.dump_task		= sched_ext_ops__dump_task,
9561 };
9562 
9563 /*
9564  * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form.
9565  * init_member, check_member, reg, unreg, etc. process kdata as the byte block
9566  * verified to match by the BUILD_BUG_ON checks in scx_init().
9567  */
9568 static struct bpf_struct_ops bpf_sched_ext_ops_cid = {
9569 	.verifier_ops = &bpf_scx_cid_verifier_ops,
9570 	.reg = bpf_scx_reg_cid,
9571 	.unreg = bpf_scx_unreg,
9572 	.check_member = bpf_scx_check_member,
9573 	.init_member = bpf_scx_init_member,
9574 	.init = bpf_scx_init,
9575 	.update = bpf_scx_update,
9576 	.validate = bpf_scx_validate,
9577 	.name = "sched_ext_ops_cid",
9578 	.owner = THIS_MODULE,
9579 	.cfi_stubs = &__bpf_ops_sched_ext_ops_cid
9580 };
9581 
9582 
9583 /********************************************************************************
9584  * System integration and init.
9585  */
9586 
9587 static void sysrq_handle_sched_ext_reset(u8 key)
9588 {
9589 	struct scx_sched *sch;
9590 
9591 	sch = rcu_dereference(scx_root);
9592 	if (likely(sch))
9593 		scx_disable(sch, SCX_EXIT_SYSRQ);
9594 	else
9595 		pr_info("sched_ext: BPF schedulers not loaded\n");
9596 }
9597 
9598 static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
9599 	.handler	= sysrq_handle_sched_ext_reset,
9600 	.help_msg	= "reset-sched-ext(S)",
9601 	.action_msg	= "Disable sched_ext and revert all tasks to CFS",
9602 	.enable_mask	= SYSRQ_ENABLE_RTNICE,
9603 };
9604 
9605 static void sysrq_handle_sched_ext_dump(u8 key)
9606 {
9607 	struct scx_exit_info ei = {
9608 		.kind		= SCX_EXIT_NONE,
9609 		.exit_cpu	= -1,
9610 		.reason		= "SysRq-D",
9611 	};
9612 	struct scx_sched *sch;
9613 
9614 	list_for_each_entry_rcu(sch, &scx_sched_all, all)
9615 		scx_dump_state(sch, &ei, 0, false);
9616 }
9617 
9618 static const struct sysrq_key_op sysrq_sched_ext_dump_op = {
9619 	.handler	= sysrq_handle_sched_ext_dump,
9620 	.help_msg	= "dump-sched-ext(D)",
9621 	.action_msg	= "Trigger sched_ext debug dump",
9622 	.enable_mask	= SYSRQ_ENABLE_RTNICE,
9623 };
9624 
9625 static bool can_skip_idle_kick(struct rq *rq)
9626 {
9627 	lockdep_assert_rq_held(rq);
9628 
9629 	/*
9630 	 * We can skip idle kicking if @rq is going to go through at least one
9631 	 * full SCX scheduling cycle before going idle. Just checking whether
9632 	 * curr is not idle is insufficient because we could be racing
9633 	 * dispatch_one() trying to pull the next task from a remote rq, which
9634 	 * may fail, and @rq may become idle afterwards.
9635 	 *
9636 	 * The race window is small and we don't and can't guarantee that @rq is
9637 	 * only kicked while idle anyway. Skip only when sure.
9638 	 */
9639 	return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_DISPATCH);
9640 }
9641 
9642 static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq,
9643 			 unsigned long *ksyncs)
9644 {
9645 	struct rq *rq = cpu_rq(cpu);
9646 	struct scx_rq *this_scx = &this_rq->scx;
9647 	struct rq_flags rf;
9648 	const struct sched_class *cur_class;
9649 	bool should_wait = false;
9650 	bool kickable;
9651 	bool preempt, preempt_lazy, wait, immediate;
9652 
9653 	rq_lock_irqsave(rq, &rf);
9654 	cur_class = rq->donor->sched_class;
9655 	preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt);
9656 	preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy);
9657 	wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait);
9658 	/*
9659 	 * Immediate preemption, waiting and a plain kick take precedence over
9660 	 * lazy preemption. The lazy request still clears the slice, so all
9661 	 * accumulated requests are served.
9662 	 */
9663 	immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick);
9664 
9665 	/*
9666 	 * During CPU hotplug, a CPU may depend on kicking itself to make
9667 	 * forward progress. Allow kicking self regardless of online state. If
9668 	 * @cpu is running a higher class task, we have no control over @cpu.
9669 	 * Skip kicking. A sub-sched lacking baseline access on @cid has no
9670 	 * business forcing a reschedule there - skip. This is the authoritative
9671 	 * cap check: ecaps is read here under @rq's lock.
9672 	 */
9673 	kickable = (cpu_online(cpu) || cpu == cpu_of(this_rq)) &&
9674 		   !sched_class_above(cur_class, &ext_sched_class);
9675 
9676 	if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) {
9677 		if (preempt || preempt_lazy) {
9678 			if (cur_class == &ext_sched_class) {
9679 				u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
9680 
9681 				if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) {
9682 					__scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1);
9683 					/* degrade to a plain, immediate kick */
9684 					immediate = true;
9685 				} else if (unlikely(!scx_set_task_slice(rq->donor, 0))) {
9686 					__scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1);
9687 				}
9688 			}
9689 			cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
9690 			cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy);
9691 		}
9692 
9693 		if (wait) {
9694 			if (cur_class == &ext_sched_class) {
9695 				cpumask_set_cpu(cpu, this_scx->cpus_to_sync);
9696 				ksyncs[cpu] = rq->scx.kick_sync;
9697 				should_wait = true;
9698 			}
9699 			cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
9700 		}
9701 
9702 		if (immediate)
9703 			resched_curr(rq);
9704 		else
9705 			scx_resched_curr_lazy(rq);
9706 	} else {
9707 		/* a kickable cpu was skipped solely for the missing caps */
9708 		if (kickable)
9709 			__scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
9710 		cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
9711 		cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy);
9712 		cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
9713 	}
9714 
9715 	scx_rq_lock_drop(rq);
9716 	rq_unlock_irqrestore(rq, &rf);
9717 
9718 	return should_wait;
9719 }
9720 
9721 static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu,
9722 				 struct rq *this_rq)
9723 {
9724 	struct rq *rq = cpu_rq(cpu);
9725 	unsigned long flags;
9726 
9727 	raw_spin_rq_lock_irqsave(rq, flags);
9728 
9729 	/* idle kicks need baseline access too, see kick_one_cpu() */
9730 	if (!can_skip_idle_kick(rq) &&
9731 	    (cpu_online(cpu) || cpu == cpu_of(this_rq))) {
9732 		if (likely(!scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)))
9733 			resched_curr(rq);
9734 		else
9735 			__scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
9736 	}
9737 
9738 	scx_rq_lock_drop(rq);
9739 	raw_spin_rq_unlock_irqrestore(rq, flags);
9740 }
9741 
9742 static void kick_cpus_irq_workfn(struct irq_work *irq_work)
9743 {
9744 	struct rq *this_rq = this_rq();
9745 	struct scx_rq *this_scx = &this_rq->scx;
9746 	struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs);
9747 	struct scx_sched_pcpu *pcpu, *tmp;
9748 	bool should_wait = false;
9749 	unsigned long *ksyncs;
9750 	s32 cpu;
9751 
9752 	/* can race with free_kick_syncs() during scheduler disable */
9753 	if (unlikely(!ksyncs_pcpu))
9754 		return;
9755 
9756 	ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs;
9757 
9758 	/*
9759 	 * Walk scheds with pending kicks on this cpu. scx_kick_cpu() adds to
9760 	 * the list under local_irq_save() and only this irq_work consumes it.
9761 	 * A plain list without locking is sufficient.
9762 	 */
9763 	list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) {
9764 		list_del_init(&pcpu->to_kick_node);
9765 
9766 		for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) {
9767 			should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs);
9768 			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick);
9769 			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
9770 		}
9771 
9772 		for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) {
9773 			kick_one_cpu_if_idle(cpu, pcpu, this_rq);
9774 			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
9775 		}
9776 	}
9777 
9778 	/*
9779 	 * Can't wait in hardirq — kick_sync can't advance, deadlocking if
9780 	 * CPUs wait for each other. Defer to kick_sync_wait_bal_cb().
9781 	 */
9782 	if (should_wait) {
9783 		raw_spin_rq_lock(this_rq);
9784 		this_scx->kick_sync_pending = true;
9785 		resched_curr(this_rq);
9786 		scx_rq_lock_drop(this_rq);
9787 		raw_spin_rq_unlock(this_rq);
9788 	}
9789 }
9790 
9791 /**
9792  * print_scx_info - print out sched_ext scheduler state
9793  * @log_lvl: the log level to use when printing
9794  * @p: target task
9795  *
9796  * If a sched_ext scheduler is enabled, print the name and state of the
9797  * scheduler. If @p is on sched_ext, print further information about the task.
9798  *
9799  * This function can be safely called on any task as long as the task_struct
9800  * itself is accessible. While safe, this function isn't synchronized and may
9801  * print out mixups or garbages of limited length.
9802  */
9803 void print_scx_info(const char *log_lvl, struct task_struct *p)
9804 {
9805 	struct scx_sched *sch;
9806 	enum scx_enable_state state = scx_enable_state();
9807 	const char *all = READ_ONCE(scx_switching_all) ? "+all" : "";
9808 	char runnable_at_buf[22] = "?";
9809 	struct sched_class *class;
9810 	unsigned long runnable_at;
9811 
9812 	guard(rcu)();
9813 
9814 	sch = scx_task_sched_rcu(p);
9815 
9816 	if (!sch)
9817 		return;
9818 
9819 	/*
9820 	 * Carefully check if the task was running on sched_ext, and then
9821 	 * carefully copy the time it's been runnable, and its state.
9822 	 */
9823 	if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) ||
9824 	    class != &ext_sched_class) {
9825 		printk("%sSched_ext: %s (%s%s)", log_lvl, sch->ops.name,
9826 		       scx_enable_state_str[state], all);
9827 		return;
9828 	}
9829 
9830 	if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at,
9831 				      sizeof(runnable_at)))
9832 		scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms",
9833 			  jiffies_delta_msecs(runnable_at, jiffies));
9834 
9835 	/* print everything onto one line to conserve console space */
9836 	printk("%sSched_ext: %s (%s%s), task: runnable_at=%s",
9837 	       log_lvl, sch->ops.name, scx_enable_state_str[state], all,
9838 	       runnable_at_buf);
9839 }
9840 
9841 static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr)
9842 {
9843 	struct scx_sched *sch;
9844 
9845 	/* track the PM cycle whether or not a root is loaded */
9846 	switch (event) {
9847 	case PM_HIBERNATION_PREPARE:
9848 	case PM_SUSPEND_PREPARE:
9849 	case PM_RESTORE_PREPARE:
9850 		scx_sub_pm_begin();
9851 		break;
9852 	case PM_POST_HIBERNATION:
9853 	case PM_POST_SUSPEND:
9854 	case PM_POST_RESTORE:
9855 		scx_sub_pm_end();
9856 		break;
9857 	}
9858 
9859 	guard(rcu)();
9860 
9861 	sch = rcu_dereference(scx_root);
9862 	if (!sch)
9863 		return NOTIFY_OK;
9864 
9865 	/*
9866 	 * SCX schedulers often have userspace components which are sometimes
9867 	 * involved in critial scheduling paths. PM operations involve freezing
9868 	 * userspace which can lead to scheduling misbehaviors including stalls.
9869 	 * Let's bypass while PM operations are in progress.
9870 	 */
9871 	switch (event) {
9872 	case PM_HIBERNATION_PREPARE:
9873 	case PM_SUSPEND_PREPARE:
9874 	case PM_RESTORE_PREPARE:
9875 		scx_bypass(sch, true);
9876 		break;
9877 	case PM_POST_HIBERNATION:
9878 	case PM_POST_SUSPEND:
9879 	case PM_POST_RESTORE:
9880 		scx_bypass(sch, false);
9881 		break;
9882 	}
9883 
9884 	return NOTIFY_OK;
9885 }
9886 
9887 static struct notifier_block scx_pm_notifier = {
9888 	.notifier_call = scx_pm_handler,
9889 };
9890 
9891 void __init init_sched_ext_class(void)
9892 {
9893 	s32 cpu, v;
9894 
9895 	/*
9896 	 * The following is to prevent the compiler from optimizing out the enum
9897 	 * definitions so that BPF scheduler implementations can use them
9898 	 * through the generated vmlinux.h.
9899 	 */
9900 	WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT |
9901 		   SCX_TG_ONLINE);
9902 
9903 	scx_idle_init_masks();
9904 
9905 	for_each_possible_cpu(cpu) {
9906 		struct rq *rq = cpu_rq(cpu);
9907 		int  n = cpu_to_node(cpu);
9908 
9909 		/* local_dsq's sch will be set during scx_root_enable() */
9910 		BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
9911 		BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
9912 #ifdef CONFIG_EXT_SUB_SCHED
9913 		scx_rescue_init(rq);
9914 #endif
9915 
9916 		INIT_LIST_HEAD(&rq->scx.runnable_list);
9917 		INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
9918 
9919 		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n));
9920 		INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick);
9921 		raw_spin_lock_init(&rq->scx.deferred_reenq_lock);
9922 		INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals);
9923 		INIT_LIST_HEAD(&rq->scx.deferred_reenq_users);
9924 		rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn);
9925 		rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn);
9926 		rq->scx.resched_irq_work = IRQ_WORK_INIT_HARD(resched_irq_workfn);
9927 
9928 		if (cpu_online(cpu))
9929 			cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE;
9930 	}
9931 
9932 	register_sysrq_key('S', &sysrq_sched_ext_reset_op);
9933 	register_sysrq_key('D', &sysrq_sched_ext_dump_op);
9934 	INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn);
9935 
9936 #ifdef CONFIG_EXT_SUB_SCHED
9937 	BUG_ON(rhashtable_init(&scx_sched_hash, &scx_sched_hash_params));
9938 #endif	/* CONFIG_EXT_SUB_SCHED */
9939 }
9940 
9941 
9942 /********************************************************************************
9943  * Helpers that can be called from the BPF scheduler.
9944  */
9945 static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)
9946 {
9947 	bool is_local = dsq_id == SCX_DSQ_LOCAL ||
9948 		(dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON;
9949 
9950 	if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
9951 		scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags);
9952 		return false;
9953 	}
9954 
9955 	if (*enq_flags & SCX_ENQ_IMMED) {
9956 		if (unlikely(!is_local)) {
9957 			scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id);
9958 			return false;
9959 		}
9960 	} else if ((sch->ops.flags & SCX_OPS_ALWAYS_ENQ_IMMED) && is_local) {
9961 		*enq_flags |= SCX_ENQ_IMMED;
9962 	}
9963 
9964 	if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) {
9965 		scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id);
9966 		return false;
9967 	}
9968 
9969 	return true;
9970 }
9971 
9972 static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p,
9973 				    u64 dsq_id, u64 *enq_flags)
9974 {
9975 	lockdep_assert_irqs_disabled();
9976 
9977 	if (unlikely(!p)) {
9978 		scx_error(sch, "called with NULL task");
9979 		return false;
9980 	}
9981 
9982 	/* see SCX_EV_INSERT_NOT_OWNED definition */
9983 	if (unlikely(!scx_task_on_sched(sch, p))) {
9984 		__scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1);
9985 		return false;
9986 	}
9987 
9988 	if (!scx_vet_enq_flags(sch, dsq_id, enq_flags))
9989 		return false;
9990 
9991 	return true;
9992 }
9993 
9994 static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,
9995 				  u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
9996 {
9997 	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
9998 	struct task_struct *ddsp_task;
9999 
10000 	ddsp_task = __this_cpu_read(direct_dispatch_task);
10001 	if (ddsp_task) {
10002 		mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags);
10003 		return;
10004 	}
10005 
10006 	if (unlikely(dspc->cursor >= sch->dsp_max_batch)) {
10007 		scx_error(sch, "dispatch buffer overflow");
10008 		return;
10009 	}
10010 
10011 	dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){
10012 		.task = p,
10013 		.qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK,
10014 		.dsq_id = dsq_id,
10015 		.slice = slice,
10016 		.vtime = vtime,
10017 		.enq_flags = enq_flags,
10018 	};
10019 }
10020 
10021 __bpf_kfunc_start_defs();
10022 
10023 /**
10024  * scx_bpf_dsq_insert___v2 - Insert a task into the FIFO queue of a DSQ
10025  * @p: task_struct to insert
10026  * @dsq_id: DSQ to insert into
10027  * @slice: duration @p can run for in nsecs, 0 to keep the current value
10028  * @enq_flags: SCX_ENQ_*
10029  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10030  *
10031  * Insert @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe to
10032  * call this function spuriously. Can be called from ops.enqueue(),
10033  * ops.select_cpu(), and ops.dispatch().
10034  *
10035  * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch
10036  * and @p must match the task being enqueued.
10037  *
10038  * When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p
10039  * will be directly inserted into the corresponding dispatch queue after
10040  * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be
10041  * inserted into the local DSQ of the CPU returned by ops.select_cpu().
10042  * @enq_flags are OR'd with the enqueue flags on the enqueue path before the
10043  * task is inserted.
10044  *
10045  * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
10046  * and this function can be called up to ops.dispatch_max_batch times to insert
10047  * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
10048  * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the
10049  * counter.
10050  *
10051  * This function doesn't have any locking restrictions and may be called under
10052  * BPF locks (in the future when BPF introduces more flexible locking).
10053  *
10054  * @p is allowed to run for @slice. The scheduling path is triggered on slice
10055  * exhaustion. If zero, the current residual slice is maintained. If
10056  * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with
10057  * scx_bpf_kick_cpu() to trigger scheduling.
10058  *
10059  * Returns %true on successful insertion, %false on failure. On the root
10060  * scheduler, %false return triggers scheduler abort and the caller doesn't need
10061  * to check the return value.
10062  */
10063 __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
10064 					 u64 slice, u64 enq_flags,
10065 					 const struct bpf_prog_aux *aux)
10066 {
10067 	struct scx_sched *sch;
10068 
10069 	guard(rcu)();
10070 	sch = scx_prog_sched(aux);
10071 	if (unlikely(!sch))
10072 		return false;
10073 
10074 	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
10075 		return false;
10076 
10077 	scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags);
10078 
10079 	return true;
10080 }
10081 
10082 /*
10083  * COMPAT: Will be removed in v6.23 along with the ___v2 suffix.
10084  */
10085 __bpf_kfunc void scx_bpf_dsq_insert(struct task_struct *p, u64 dsq_id,
10086 				    u64 slice, u64 enq_flags,
10087 				    const struct bpf_prog_aux *aux)
10088 {
10089 	scx_bpf_dsq_insert___v2(p, dsq_id, slice, enq_flags, aux);
10090 }
10091 
10092 static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,
10093 				 u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
10094 {
10095 	if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
10096 		return false;
10097 
10098 	scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ);
10099 
10100 	return true;
10101 }
10102 
10103 struct scx_bpf_dsq_insert_vtime_args {
10104 	/* @p can't be packed together as KF_RCU is not transitive */
10105 	u64			dsq_id;
10106 	u64			slice;
10107 	u64			vtime;
10108 	u64			enq_flags;
10109 };
10110 
10111 /**
10112  * __scx_bpf_dsq_insert_vtime - Arg-wrapped vtime DSQ insertion
10113  * @p: task_struct to insert
10114  * @args: struct containing the rest of the arguments
10115  *       @args->dsq_id: DSQ to insert into
10116  *       @args->slice: duration @p can run for in nsecs, 0 to keep the current value
10117  *       @args->vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
10118  *       @args->enq_flags: SCX_ENQ_*
10119  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10120  *
10121  * Wrapper kfunc that takes arguments via struct to work around BPF's 5 argument
10122  * limit. BPF programs should use scx_bpf_dsq_insert_vtime() which is provided
10123  * as an inline wrapper in common.bpf.h.
10124  *
10125  * Insert @p into the vtime priority queue of the DSQ identified by
10126  * @args->dsq_id. Tasks queued into the priority queue are ordered by
10127  * @args->vtime. All other aspects are identical to scx_bpf_dsq_insert().
10128  *
10129  * @args->vtime ordering is according to time_before64() which considers
10130  * wrapping. A numerically larger vtime may indicate an earlier position in the
10131  * ordering and vice-versa. vtime is a rolling cursor and values used for
10132  * ordering within a given DSQ should stay less than 2^63 apart for
10133  * time_before64() ordering to remain well-defined.
10134  *
10135  * A DSQ can only be used as a FIFO or priority queue at any given time and this
10136  * function must not be called on a DSQ which already has one or more FIFO tasks
10137  * queued and vice-versa. Also, the built-in DSQs (SCX_DSQ_LOCAL and
10138  * SCX_DSQ_GLOBAL) cannot be used as priority queues.
10139  *
10140  * Returns %true on successful insertion, %false on failure. On the root
10141  * scheduler, %false return triggers scheduler abort and the caller doesn't need
10142  * to check the return value.
10143  */
10144 __bpf_kfunc bool
10145 __scx_bpf_dsq_insert_vtime(struct task_struct *p,
10146 			   struct scx_bpf_dsq_insert_vtime_args *args,
10147 			   const struct bpf_prog_aux *aux)
10148 {
10149 	struct scx_sched *sch;
10150 
10151 	guard(rcu)();
10152 
10153 	sch = scx_prog_sched(aux);
10154 	if (unlikely(!sch))
10155 		return false;
10156 
10157 	return scx_dsq_insert_vtime(sch, p, args->dsq_id, args->slice,
10158 				    args->vtime, args->enq_flags);
10159 }
10160 
10161 /*
10162  * COMPAT: Will be removed in v6.23.
10163  */
10164 __bpf_kfunc void scx_bpf_dsq_insert_vtime(struct task_struct *p, u64 dsq_id,
10165 					  u64 slice, u64 vtime, u64 enq_flags)
10166 {
10167 	struct scx_sched *sch;
10168 
10169 	guard(rcu)();
10170 
10171 	sch = rcu_dereference(scx_root);
10172 	if (unlikely(!sch))
10173 		return;
10174 
10175 #ifdef CONFIG_EXT_SUB_SCHED
10176 	/*
10177 	 * Disallow if any sub-scheds are attached. There is no way to tell
10178 	 * which scheduler called us, so error out @p's scheduler -- read it
10179 	 * under RCU as @p's locks aren't necessarily held here. @p may be a
10180 	 * task past sched_ext_dead() or an idle task, in which case its
10181 	 * scheduler can't be determined and there is nothing obviously wrong
10182 	 * to report; just refuse the call.
10183 	 */
10184 	if (unlikely(!list_empty(&sch->children))) {
10185 		struct scx_sched *tsch = scx_task_sched_rcu(p);
10186 
10187 		if (tsch)
10188 			scx_error(tsch, "__scx_bpf_dsq_insert_vtime() must be used");
10189 		return;
10190 	}
10191 #endif
10192 
10193 	scx_dsq_insert_vtime(sch, p, dsq_id, slice, vtime, enq_flags);
10194 }
10195 
10196 __bpf_kfunc_end_defs();
10197 
10198 BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch)
10199 BTF_ID_FLAGS(func, scx_bpf_dsq_insert, KF_IMPLICIT_ARGS | KF_RCU)
10200 BTF_ID_FLAGS(func, scx_bpf_dsq_insert___v2, KF_IMPLICIT_ARGS | KF_RCU)
10201 BTF_ID_FLAGS(func, __scx_bpf_dsq_insert_vtime, KF_IMPLICIT_ARGS | KF_RCU)
10202 BTF_ID_FLAGS(func, scx_bpf_dsq_insert_vtime, KF_RCU)
10203 BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch)
10204 
10205 static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
10206 	.owner			= THIS_MODULE,
10207 	.set			= &scx_kfunc_ids_enqueue_dispatch,
10208 	.filter			= scx_kfunc_context_filter,
10209 };
10210 
10211 /*
10212  * Lock @rq from a kfunc called either from ops.dispatch(), which holds the
10213  * dispatched rq's lock, or from a context holding no rq lock. Under core
10214  * scheduling, dispatch can run for a sibling rq, so the locked rq is not
10215  * necessarily this CPU's. Return the rq locked on entry, NULL if none.
10216  */
10217 static struct rq *kf_switch_rq_lock(struct rq *rq, unsigned long *flags)
10218 {
10219 	struct rq *entry_rq;
10220 
10221 	local_irq_save(*flags);
10222 
10223 	entry_rq = scx_locked_rq();
10224 	if (entry_rq) {
10225 		if (entry_rq != rq)
10226 			switch_rq_lock(entry_rq, rq);
10227 	} else {
10228 		raw_spin_rq_lock(rq);
10229 	}
10230 	return entry_rq;
10231 }
10232 
10233 /* undo kf_switch_rq_lock(), @rq is the rq locked now */
10234 static void kf_restore_rq_lock(struct rq *rq, struct rq *entry_rq, unsigned long flags)
10235 {
10236 	if (entry_rq) {
10237 		if (entry_rq != rq)
10238 			switch_rq_lock(rq, entry_rq);
10239 	} else {
10240 		scx_rq_lock_drop(rq);
10241 		raw_spin_rq_unlock_irqrestore(rq, flags);
10242 	}
10243 }
10244 
10245 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
10246 			 struct task_struct *p, u64 dsq_id, u64 enq_flags,
10247 			 bool priq)
10248 {
10249 	struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;
10250 	struct scx_sched *sch;
10251 	struct rq *p_rq, *src_rq, *locked_rq;
10252 	bool dispatched = false;
10253 	unsigned long flags;
10254 
10255 	/*
10256 	 * The verifier considers an iterator slot initialized on any
10257 	 * KF_ITER_NEW return, so a BPF program may legally reach here after
10258 	 * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL.
10259 	 */
10260 	if (unlikely(!src_dsq))
10261 		return false;
10262 
10263 	sch = src_dsq->sched;
10264 
10265 	if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
10266 		return false;
10267 
10268 	/* internal bit, can only go in after @enq_flags is vetted */
10269 	if (priq)
10270 		enq_flags |= SCX_ENQ_DSQ_PRIQ;
10271 
10272 	/*
10273 	 * If the BPF scheduler keeps calling this function repeatedly, it can
10274 	 * cause similar live-lock conditions as scx_consume_dispatch_q().
10275 	 */
10276 	if (unlikely(READ_ONCE(sch->aborting)))
10277 		return false;
10278 
10279 	src_rq = task_rq(p);
10280 	locked_rq = kf_switch_rq_lock(src_rq, &flags);
10281 
10282 	p_rq = src_rq;
10283 	raw_spin_lock(&src_dsq->lock);
10284 
10285 	/* did someone else get to it while we dropped the locks? */
10286 	if (nldsq_cursor_lost_task(&kit->cursor, src_rq, src_dsq, p)) {
10287 		raw_spin_unlock(&src_dsq->lock);
10288 		goto out;
10289 	}
10290 
10291 	/*
10292 	 * @p has been on $src_dsq and can't move anymore. If @p is not on @sch,
10293 	 * the caller didn't have authority over @p at the time of the call.
10294 	 */
10295 	if (unlikely(!scx_task_on_sched(sch, p))) {
10296 		scx_error(sch, "scx_bpf_dsq_move[_vtime]() on %s[%d] but the task belongs to a different scheduler",
10297 			  p->comm, p->pid);
10298 		raw_spin_unlock(&src_dsq->lock);
10299 		goto out;
10300 	}
10301 
10302 	/* @p is still on $src_dsq and stable, determine the destination */
10303 	dst_dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, task_cpu(p));
10304 
10305 	/*
10306 	 * Apply vtime and slice updates before moving. @p is still on $src_dsq
10307 	 * with both $src_dsq and its task_rq locked, satisfying the write
10308 	 * rules, and the PRIQ insertion into $dst_dsq reads the new vtime.
10309 	 */
10310 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
10311 		p->scx.dsq_vtime = kit->vtime;
10312 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)
10313 		scx_set_task_slice(p, kit->slice);
10314 
10315 	/* execute move */
10316 	p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq);
10317 	dispatched = true;
10318 out:
10319 	kf_restore_rq_lock(p_rq, locked_rq, flags);
10320 
10321 	kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE |
10322 			       __SCX_DSQ_ITER_HAS_VTIME);
10323 	return dispatched;
10324 }
10325 
10326 __bpf_kfunc_start_defs();
10327 
10328 /**
10329  * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots
10330  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10331  *
10332  * Can only be called from ops.dispatch().
10333  */
10334 __bpf_kfunc u32 scx_bpf_dispatch_nr_slots(const struct bpf_prog_aux *aux)
10335 {
10336 	struct scx_sched *sch;
10337 
10338 	guard(rcu)();
10339 
10340 	sch = scx_prog_sched(aux);
10341 	if (unlikely(!sch))
10342 		return 0;
10343 
10344 	return sch->dsp_max_batch - __this_cpu_read(sch->pcpu->dsp_ctx.cursor);
10345 }
10346 
10347 /**
10348  * scx_bpf_dispatch_cancel - Cancel the latest dispatch
10349  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10350  *
10351  * Cancel the latest dispatch. Can be called multiple times to cancel further
10352  * dispatches. Can only be called from ops.dispatch().
10353  */
10354 __bpf_kfunc void scx_bpf_dispatch_cancel(const struct bpf_prog_aux *aux)
10355 {
10356 	struct scx_sched *sch;
10357 	struct scx_dsp_ctx *dspc;
10358 
10359 	guard(rcu)();
10360 
10361 	sch = scx_prog_sched(aux);
10362 	if (unlikely(!sch))
10363 		return;
10364 
10365 	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
10366 
10367 	if (dspc->cursor > 0)
10368 		dspc->cursor--;
10369 	else
10370 		scx_error(sch, "dispatch buffer underflow");
10371 }
10372 
10373 /**
10374  * scx_bpf_dsq_move_to_local___v2 - move a task from a DSQ to the current CPU's local DSQ
10375  * @dsq_id: DSQ to move task from. Must be a user-created DSQ
10376  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10377  * @enq_flags: %SCX_ENQ_*
10378  *
10379  * Move a task from the non-local DSQ identified by @dsq_id to the current CPU's
10380  * local DSQ for execution with @enq_flags applied. Can only be called from
10381  * ops.dispatch().
10382  *
10383  * Built-in DSQs (%SCX_DSQ_GLOBAL and %SCX_DSQ_LOCAL*) are not supported as
10384  * sources. Local DSQs support reenqueueing (a task can be picked up for
10385  * execution, dequeued for property changes, or reenqueued), but the BPF
10386  * scheduler cannot directly iterate or move tasks from them. %SCX_DSQ_GLOBAL
10387  * is similar but also doesn't support reenqueueing, as it maps to multiple
10388  * per-node DSQs making the scope difficult to define; this may change in the
10389  * future.
10390  *
10391  * This function flushes the in-flight dispatches from scx_bpf_dsq_insert()
10392  * before trying to move from the specified DSQ. It may also grab rq locks and
10393  * thus can't be called under any BPF locks.
10394  *
10395  * Returns %true if a task has been moved, %false if there isn't any task to
10396  * move.
10397  */
10398 __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,
10399 						const struct bpf_prog_aux *aux)
10400 {
10401 	struct scx_dispatch_q *dsq;
10402 	struct scx_sched *sch;
10403 	struct scx_dsp_ctx *dspc;
10404 
10405 	guard(rcu)();
10406 
10407 	sch = scx_prog_sched(aux);
10408 	if (unlikely(!sch))
10409 		return false;
10410 
10411 	if (!scx_vet_enq_flags(sch, SCX_DSQ_LOCAL, &enq_flags))
10412 		return false;
10413 
10414 	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
10415 
10416 	scx_flush_dispatch_buf(sch, dspc->rq);
10417 
10418 	dsq = find_user_dsq(sch, dsq_id);
10419 	if (unlikely(!dsq)) {
10420 		scx_error(sch, "invalid DSQ ID 0x%016llx", dsq_id);
10421 		return false;
10422 	}
10423 
10424 	if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) {
10425 		/*
10426 		 * A successfully consumed task can be dequeued before it starts
10427 		 * running while the CPU is trying to migrate other dispatched
10428 		 * tasks. Bump nr_tasks to tell dispatch_one() to retry on empty
10429 		 * local DSQ.
10430 		 */
10431 		dspc->nr_tasks++;
10432 		return true;
10433 	} else {
10434 		return false;
10435 	}
10436 }
10437 
10438 /*
10439  * COMPAT: ___v2 was introduced in v7.1. Remove this and ___v2 tag in the future.
10440  */
10441 __bpf_kfunc bool scx_bpf_dsq_move_to_local(u64 dsq_id, const struct bpf_prog_aux *aux)
10442 {
10443 	return scx_bpf_dsq_move_to_local___v2(dsq_id, 0, aux);
10444 }
10445 
10446 /**
10447  * scx_bpf_dsq_move_set_slice - Override slice when moving between DSQs
10448  * @it__iter: DSQ iterator in progress
10449  * @slice: duration the moved task can run for in nsecs
10450  *
10451  * Override the slice of the next task that will be moved from @it__iter using
10452  * scx_bpf_dsq_move[_vtime](). If this function is not called, the previous
10453  * slice duration is kept.
10454  */
10455 __bpf_kfunc void scx_bpf_dsq_move_set_slice(struct bpf_iter_scx_dsq *it__iter,
10456 					    u64 slice)
10457 {
10458 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
10459 
10460 	kit->slice = slice;
10461 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE;
10462 }
10463 
10464 /**
10465  * scx_bpf_dsq_move_set_vtime - Override vtime when moving between DSQs
10466  * @it__iter: DSQ iterator in progress
10467  * @vtime: task's ordering inside the vtime-sorted queue of the target DSQ
10468  *
10469  * Override the vtime of the next task that will be moved from @it__iter using
10470  * scx_bpf_dsq_move_vtime(). If this function is not called, the previous slice
10471  * vtime is kept. If scx_bpf_dsq_move() is used to dispatch the next task, the
10472  * override is ignored and cleared.
10473  */
10474 __bpf_kfunc void scx_bpf_dsq_move_set_vtime(struct bpf_iter_scx_dsq *it__iter,
10475 					    u64 vtime)
10476 {
10477 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
10478 
10479 	kit->vtime = vtime;
10480 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME;
10481 }
10482 
10483 /**
10484  * scx_bpf_dsq_move - Move a task from DSQ iteration to a DSQ
10485  * @it__iter: DSQ iterator in progress
10486  * @p: task to transfer
10487  * @dsq_id: DSQ to move @p to
10488  * @enq_flags: SCX_ENQ_*
10489  *
10490  * Transfer @p which is on the DSQ currently iterated by @it__iter to the DSQ
10491  * specified by @dsq_id. All DSQs - local DSQs, global DSQ and user DSQs - can
10492  * be the destination.
10493  *
10494  * For the transfer to be successful, @p must still be on the DSQ and have been
10495  * queued before the DSQ iteration started. This function doesn't care whether
10496  * @p was obtained from the DSQ iteration. @p just has to be on the DSQ and have
10497  * been queued before the iteration started.
10498  *
10499  * @p's slice is kept by default. Use scx_bpf_dsq_move_set_slice() to update.
10500  *
10501  * Can be called from ops.dispatch() or any BPF context which doesn't hold a rq
10502  * lock (e.g. BPF timers or SYSCALL programs).
10503  *
10504  * Returns %true if @p has been consumed, %false if @p had already been
10505  * consumed, dequeued, or, for sub-scheds, @dsq_id points to a disallowed local
10506  * DSQ.
10507  */
10508 __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
10509 				  struct task_struct *p, u64 dsq_id,
10510 				  u64 enq_flags)
10511 {
10512 	return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
10513 			    p, dsq_id, enq_flags, false);
10514 }
10515 
10516 /**
10517  * scx_bpf_dsq_move_vtime - Move a task from DSQ iteration to a PRIQ DSQ
10518  * @it__iter: DSQ iterator in progress
10519  * @p: task to transfer
10520  * @dsq_id: DSQ to move @p to
10521  * @enq_flags: SCX_ENQ_*
10522  *
10523  * Transfer @p which is on the DSQ currently iterated by @it__iter to the
10524  * priority queue of the DSQ specified by @dsq_id. The destination must be a
10525  * user DSQ as only user DSQs support priority queue.
10526  *
10527  * @p's slice and vtime are kept by default. Use scx_bpf_dsq_move_set_slice()
10528  * and scx_bpf_dsq_move_set_vtime() to update.
10529  *
10530  * All other aspects are identical to scx_bpf_dsq_move(). See
10531  * scx_bpf_dsq_insert_vtime() for more information on @vtime.
10532  */
10533 __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter,
10534 					struct task_struct *p, u64 dsq_id,
10535 					u64 enq_flags)
10536 {
10537 	return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
10538 			    p, dsq_id, enq_flags, true);
10539 }
10540 
10541 __bpf_kfunc_end_defs();
10542 
10543 BTF_KFUNCS_START(scx_kfunc_ids_dispatch)
10544 BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots, KF_IMPLICIT_ARGS)
10545 BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel, KF_IMPLICIT_ARGS)
10546 BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local, KF_IMPLICIT_ARGS)
10547 BTF_ID_FLAGS(func, scx_bpf_dsq_move_to_local___v2, KF_IMPLICIT_ARGS)
10548 /* scx_bpf_dsq_move*() also in scx_kfunc_ids_unlocked: callable from unlocked contexts */
10549 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
10550 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
10551 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
10552 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
10553 /* also in scx_kfunc_ids_unlocked, same contexts as scx_bpf_dsq_move() */
10554 BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU)
10555 #ifdef CONFIG_EXT_SUB_SCHED
10556 BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS)
10557 #endif
10558 BTF_KFUNCS_END(scx_kfunc_ids_dispatch)
10559 
10560 static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = {
10561 	.owner			= THIS_MODULE,
10562 	.set			= &scx_kfunc_ids_dispatch,
10563 	.filter			= scx_kfunc_context_filter,
10564 };
10565 
10566 __bpf_kfunc_start_defs();
10567 
10568 /**
10569  * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
10570  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10571  *
10572  * Iterate over all of the tasks currently enqueued on the local DSQ of the
10573  * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of
10574  * processed tasks. Can only be called from ops.cpu_release().
10575  */
10576 __bpf_kfunc u32 scx_bpf_reenqueue_local(const struct bpf_prog_aux *aux)
10577 {
10578 	struct scx_sched *sch;
10579 	struct rq *rq;
10580 
10581 	guard(rcu)();
10582 	sch = scx_prog_sched(aux);
10583 	if (unlikely(!sch))
10584 		return 0;
10585 
10586 	rq = cpu_rq(smp_processor_id());
10587 	lockdep_assert_rq_held(rq);
10588 
10589 	return reenq_local(sch, rq, SCX_REENQ_ANY);
10590 }
10591 
10592 __bpf_kfunc_end_defs();
10593 
10594 BTF_KFUNCS_START(scx_kfunc_ids_cpu_release)
10595 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local, KF_IMPLICIT_ARGS)
10596 BTF_KFUNCS_END(scx_kfunc_ids_cpu_release)
10597 
10598 static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = {
10599 	.owner			= THIS_MODULE,
10600 	.set			= &scx_kfunc_ids_cpu_release,
10601 	.filter			= scx_kfunc_context_filter,
10602 };
10603 
10604 __bpf_kfunc_start_defs();
10605 
10606 /**
10607  * scx_bpf_create_dsq - Create a custom DSQ
10608  * @dsq_id: DSQ to create
10609  * @node: NUMA node to allocate from
10610  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10611  *
10612  * Create a custom DSQ identified by @dsq_id. Can be called from any sleepable
10613  * scx callback, and any BPF_PROG_TYPE_SYSCALL prog.
10614  */
10615 __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_aux *aux)
10616 {
10617 	struct scx_dispatch_q *dsq;
10618 	struct scx_sched *sch;
10619 	s32 ret;
10620 
10621 	if (unlikely(node >= (int)nr_node_ids ||
10622 		     (node < 0 && node != NUMA_NO_NODE)))
10623 		return -EINVAL;
10624 
10625 	if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN))
10626 		return -EINVAL;
10627 
10628 	dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node);
10629 	if (!dsq)
10630 		return -ENOMEM;
10631 
10632 	/*
10633 	 * scx_init_dsq() must be called in GFP_KERNEL context. Init it with
10634 	 * NULL @sch and update afterwards.
10635 	 */
10636 	ret = scx_init_dsq(dsq, dsq_id, NULL);
10637 	if (ret) {
10638 		kfree(dsq);
10639 		return ret;
10640 	}
10641 
10642 	rcu_read_lock();
10643 
10644 	sch = scx_prog_sched(aux);
10645 	if (sch) {
10646 		dsq->sched = sch;
10647 		ret = rhashtable_lookup_insert_fast(&sch->dsq_hash, &dsq->hash_node,
10648 						    dsq_hash_params);
10649 	} else {
10650 		ret = -ENODEV;
10651 	}
10652 
10653 	rcu_read_unlock();
10654 	if (ret) {
10655 		exit_dsq(dsq);
10656 		kfree(dsq);
10657 	}
10658 	return ret;
10659 }
10660 
10661 __bpf_kfunc_end_defs();
10662 
10663 BTF_KFUNCS_START(scx_kfunc_ids_unlocked)
10664 BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_IMPLICIT_ARGS | KF_SLEEPABLE)
10665 /* also in scx_kfunc_ids_dispatch: also callable from ops.dispatch() */
10666 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU)
10667 BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU)
10668 BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU)
10669 BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU)
10670 BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU)
10671 /* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */
10672 BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU)
10673 BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)
10674 BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU)
10675 BTF_KFUNCS_END(scx_kfunc_ids_unlocked)
10676 
10677 static const struct btf_kfunc_id_set scx_kfunc_set_unlocked = {
10678 	.owner			= THIS_MODULE,
10679 	.set			= &scx_kfunc_ids_unlocked,
10680 	.filter			= scx_kfunc_context_filter,
10681 };
10682 
10683 __bpf_kfunc_start_defs();
10684 
10685 /**
10686  * scx_bpf_task_set_slice - Set task's time slice
10687  * @p: task of interest
10688  * @slice: time slice to set in nsecs
10689  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10690  *
10691  * Set @p's time slice. @p must be on the calling scheduler. The value is
10692  * applied whether or not the caller holds @p's rq lock - see the slice write
10693  * rules above for the ownership model.
10694  *
10695  * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on
10696  * @p's cpu, and never on a kernel slice, see %SCX_TASK_KERNEL_SLICE. Shortening
10697  * is honored unless @p is protected, see %SCX_TASK_PROTECTED, which refuses
10698  * every write. A refused write is counted in %SCX_EV_SLICE_DENIED. On the
10699  * stashed path the slice is packed into an atomic64_t with the scheduler id and
10700  * a flag bit, so a slice too large to fit is clamped and counted in
10701  * %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved.
10702  *
10703  * Return %true on success, %false if @p is not on the calling scheduler.
10704  */
10705 __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
10706 					const struct bpf_prog_aux *aux)
10707 {
10708 	struct scx_sched *sch;
10709 	struct rq *locked_rq;
10710 
10711 	guard(rcu)();
10712 	sch = scx_prog_sched(aux);
10713 	if (unlikely(!sch || !scx_task_on_sched(sch, p)))
10714 		return false;
10715 
10716 	/*
10717 	 * Directly write only when we hold the lock of the rq @p is queued on or
10718 	 * provides the current scheduling context for. Under proxy execution,
10719 	 * rq->donor owns and consumes the slice while rq->curr executes on its
10720 	 * behalf. See the slice write rules above.
10721 	 *
10722 	 * While @p is queued on a user DSQ or in the BPF scheduler,
10723 	 * synchronization is the scheduler's responsibility. This write can
10724 	 * race a concurrent dispatch's commit, see apply_slice_vtime().
10725 	 *
10726 	 * Making this kfunc always go through the oob stash would leave the
10727 	 * commit as the only direct writer and close the race, but that would
10728 	 * require two more oob application points - the dispatch keep-prev test
10729 	 * and the tick-time expiry check.
10730 	 */
10731 	locked_rq = scx_locked_rq();
10732 	if (!locked_rq ||
10733 	    (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) &&
10734 	     !task_current_donor(locked_rq, p))) {
10735 		set_task_slice_oob(sch, p, slice);
10736 		return true;
10737 	}
10738 
10739 	/*
10740 	 * @p's rq is locked, so apply now. An extension needs baseline access
10741 	 * and is refused on a kernel slice.
10742 	 */
10743 	if (slice > p->scx.slice &&
10744 	    unlikely(scx_task_has_kernel_slice(p) ||
10745 		     scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) {
10746 		__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
10747 		return true;
10748 	}
10749 
10750 	if (unlikely(!scx_set_task_slice(p, slice)))
10751 		__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
10752 
10753 	return true;
10754 }
10755 
10756 /**
10757  * scx_bpf_task_set_dsq_vtime - Set task's virtual time for DSQ ordering
10758  * @p: task of interest
10759  * @vtime: virtual time to set
10760  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10761  *
10762  * Set @p's virtual time to @vtime. Returns %true on success, %false if the
10763  * calling scheduler doesn't have authority over @p.
10764  */
10765 __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime,
10766 					    const struct bpf_prog_aux *aux)
10767 {
10768 	struct scx_sched *sch;
10769 
10770 	guard(rcu)();
10771 	sch = scx_prog_sched(aux);
10772 	if (unlikely(!sch || !scx_task_on_sched(sch, p)))
10773 		return false;
10774 
10775 	p->scx.dsq_vtime = vtime;
10776 	return true;
10777 }
10778 
10779 /**
10780  * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode
10781  * @p: task of interest
10782  * @lazy: whether slice expiry should request lazy rescheduling
10783  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10784  *
10785  * Choose whether depletion of @p's slice at the scheduler tick requests lazy or
10786  * immediate rescheduling. This is a persistent per-task policy and does not
10787  * itself clear @p's slice or request rescheduling, unlike the
10788  * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations.
10789  *
10790  * With lazy rescheduling enabled, a task interrupted in user space still
10791  * reschedules before returning to user space. A task interrupted in the kernel
10792  * may continue until its next return to user space or the next scheduler tick,
10793  * which promotes the request. If lazy preemption is disabled at runtime, the
10794  * request behaves like an immediate reschedule.
10795  *
10796  * @p must be on the calling scheduler.
10797  *
10798  * Return %true on success, %false if @p is not on the calling scheduler.
10799  */
10800 __bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy,
10801 					       const struct bpf_prog_aux *aux)
10802 {
10803 	struct scx_sched *sch;
10804 
10805 	/*
10806 	 * clang's register locations for @lazy make pahole 1.32 drop this kfunc
10807 	 * from BTF. Keep @lazy in memory so its location names no register.
10808 	 */
10809 	barrier_data(&lazy);
10810 	guard(rcu)();
10811 	sch = scx_prog_sched(aux);
10812 	if (unlikely(!sch || !scx_task_on_sched(sch, p)))
10813 		return false;
10814 
10815 	WRITE_ONCE(p->scx.lazy_resched, lazy);
10816 	return true;
10817 }
10818 
10819 void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
10820 {
10821 	struct scx_sched_pcpu *pcpu;
10822 	struct rq *this_rq;
10823 	unsigned long irq_flags;
10824 
10825 	if (!scx_kf_allowed_ctx(sch))
10826 		return;
10827 	if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) {
10828 		scx_error(sch, "invalid kick flags 0x%llx", flags);
10829 		return;
10830 	}
10831 
10832 	local_irq_save(irq_flags);
10833 
10834 	this_rq = this_rq();
10835 	pcpu = this_cpu_ptr(sch->pcpu);
10836 
10837 	/*
10838 	 * While bypassing for PM ops, IRQ handling may not be online which can
10839 	 * lead to irq_work_queue() malfunction such as infinite busy wait for
10840 	 * IRQ status update. Suppress kicking.
10841 	 */
10842 	if (scx_bypassing(sch, cpu_of(this_rq)))
10843 		goto out;
10844 
10845 	/*
10846 	 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting
10847 	 * rq locks. We can probably be smarter and avoid bouncing if called
10848 	 * from ops which don't hold a rq lock.
10849 	 *
10850 	 * The kick masks are owned by @sch->pcpu, so that a preempt kick can be
10851 	 * attributed to @sch.
10852 	 */
10853 	if (flags & SCX_KICK_IDLE) {
10854 		struct rq *target_rq = cpu_rq(cpu);
10855 
10856 		if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY |
10857 				     SCX_KICK_WAIT)))
10858 			scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE");
10859 
10860 		if (raw_spin_rq_trylock(target_rq)) {
10861 			if (can_skip_idle_kick(target_rq)) {
10862 				scx_rq_lock_drop(target_rq);
10863 				raw_spin_rq_unlock(target_rq);
10864 				goto out;
10865 			}
10866 			scx_rq_lock_drop(target_rq);
10867 			raw_spin_rq_unlock(target_rq);
10868 		}
10869 		cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle);
10870 	} else {
10871 		/*
10872 		 * Accumulate requests and resolve their precedence at delivery.
10873 		 */
10874 		if (flags & SCX_KICK_PREEMPT)
10875 			cpumask_set_cpu(cpu, pcpu->cpus_to_preempt);
10876 		if (flags & SCX_KICK_PREEMPT_LAZY)
10877 			cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy);
10878 		if (flags & SCX_KICK_WAIT)
10879 			cpumask_set_cpu(cpu, pcpu->cpus_to_wait);
10880 		if (!(flags & SCX_KICK_PREEMPT_LAZY))
10881 			cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
10882 	}
10883 
10884 	if (list_empty(&pcpu->to_kick_node))
10885 		list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick);
10886 	irq_work_queue(&this_rq->scx.kick_cpus_irq_work);
10887 out:
10888 	local_irq_restore(irq_flags);
10889 }
10890 
10891 /**
10892  * scx_bpf_kick_cpu - Trigger reschedule on a CPU
10893  * @cpu: cpu to kick
10894  * @flags: %SCX_KICK_* flags
10895  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10896  *
10897  * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or
10898  * trigger rescheduling on a busy CPU. This can be called from any online
10899  * scx_ops operation and the actual kicking is performed asynchronously through
10900  * an irq work.
10901  */
10902 __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux *aux)
10903 {
10904 	struct scx_sched *sch;
10905 
10906 	guard(rcu)();
10907 	sch = scx_prog_sched(aux);
10908 	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
10909 		scx_kick_cpu(sch, cpu, flags);
10910 }
10911 
10912 /**
10913  * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid
10914  * @cid: cid to kick
10915  * @flags: %SCX_KICK_* flags
10916  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10917  *
10918  * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the
10919  * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a
10920  * kick is dropped if the caller lacks baseline access on @cid, and a
10921  * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain
10922  * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its
10923  * subtree.
10924  */
10925 __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux)
10926 {
10927 	struct scx_sched *sch;
10928 	s32 cpu;
10929 
10930 	guard(rcu)();
10931 	sch = scx_prog_sched(aux);
10932 	if (unlikely(!sch))
10933 		return;
10934 	cpu = scx_cid_to_cpu(sch, cid);
10935 	if (cpu < 0)
10936 		return;
10937 	scx_kick_cpu(sch, cpu, flags);
10938 }
10939 
10940 /**
10941  * scx_bpf_dsq_nr_queued - Return the number of queued tasks
10942  * @dsq_id: id of the DSQ
10943  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10944  *
10945  * Return the number of tasks in the DSQ matching @dsq_id. If not found,
10946  * -%ENOENT is returned.
10947  *
10948  * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
10949  * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
10950  * or the calling CPU's when no rq is locked.
10951  */
10952 __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux)
10953 {
10954 	struct scx_sched *sch;
10955 	struct scx_dispatch_q *dsq;
10956 	s32 ret;
10957 
10958 	preempt_disable();
10959 
10960 	sch = scx_prog_sched(aux);
10961 	if (unlikely(!sch)) {
10962 		ret = -ENODEV;
10963 		goto out;
10964 	}
10965 
10966 	if (dsq_id == SCX_DSQ_LOCAL) {
10967 		ret = READ_ONCE((scx_locked_rq() ?: this_rq())->scx.local_dsq.nr);
10968 		goto out;
10969 	} else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
10970 		s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
10971 
10972 		if (scx_cpu_valid(sch, cpu, NULL)) {
10973 			ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr);
10974 			goto out;
10975 		}
10976 	} else {
10977 		dsq = find_user_dsq(sch, dsq_id);
10978 		if (dsq) {
10979 			ret = READ_ONCE(dsq->nr);
10980 			goto out;
10981 		}
10982 	}
10983 	ret = -ENOENT;
10984 out:
10985 	preempt_enable();
10986 	return ret;
10987 }
10988 
10989 /**
10990  * scx_bpf_destroy_dsq - Destroy a custom DSQ
10991  * @dsq_id: DSQ to destroy
10992  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
10993  *
10994  * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
10995  * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
10996  * empty and no further tasks are dispatched to it. Ignored if called on a DSQ
10997  * which doesn't exist. Can be called from any online scx_ops operations.
10998  */
10999 __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
11000 {
11001 	struct scx_sched *sch;
11002 
11003 	guard(rcu)();
11004 	sch = scx_prog_sched(aux);
11005 	if (unlikely(!sch))
11006 		return;
11007 
11008 	if (!scx_kf_allowed_ctx(sch))
11009 		return;
11010 
11011 	destroy_dsq(sch, dsq_id);
11012 }
11013 
11014 /**
11015  * bpf_iter_scx_dsq_new - Create a DSQ iterator
11016  * @it: iterator to initialize
11017  * @dsq_id: DSQ to iterate
11018  * @flags: %SCX_DSQ_ITER_*
11019  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11020  *
11021  * Initialize BPF iterator @it which can be used with bpf_for_each() to walk
11022  * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes
11023  * tasks which are already queued when this function is invoked.
11024  */
11025 __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
11026 				     u64 flags, const struct bpf_prog_aux *aux)
11027 {
11028 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;
11029 	struct scx_sched *sch;
11030 
11031 	BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) >
11032 		     sizeof(struct bpf_iter_scx_dsq));
11033 	BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) !=
11034 		     __alignof__(struct bpf_iter_scx_dsq));
11035 	BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS &
11036 		     ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1));
11037 
11038 	/*
11039 	 * next() and destroy() will be called regardless of the return value.
11040 	 * Always clear $kit->dsq.
11041 	 */
11042 	kit->dsq = NULL;
11043 
11044 	sch = scx_prog_sched(aux);
11045 	if (unlikely(!sch))
11046 		return -ENODEV;
11047 
11048 	if (!scx_kf_allowed_ctx(sch))
11049 		return -EDEADLK;
11050 
11051 	if (flags & ~__SCX_DSQ_ITER_USER_FLAGS)
11052 		return -EINVAL;
11053 
11054 	kit->dsq = find_user_dsq(sch, dsq_id);
11055 	if (!kit->dsq)
11056 		return -ENOENT;
11057 
11058 	kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, kit->dsq, flags);
11059 
11060 	return 0;
11061 }
11062 
11063 /**
11064  * bpf_iter_scx_dsq_next - Progress a DSQ iterator
11065  * @it: iterator to progress
11066  *
11067  * Return the next task. See bpf_iter_scx_dsq_new().
11068  */
11069 __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it)
11070 {
11071 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;
11072 	struct task_struct *p;
11073 
11074 	if (!kit->dsq || (kit->cursor.flags & __SCX_DSQ_ITER_DRAINED))
11075 		return NULL;
11076 
11077 	guard(raw_spinlock_irqsave)(&kit->dsq->lock);
11078 
11079 	p = nldsq_cursor_next_task(&kit->cursor, kit->dsq);
11080 	/*
11081 	 * nldsq_cursor_next_task() unlinks the cursor and starts from the DSQ
11082 	 * head when it is called again.  The verifier relies on an iterator
11083 	 * which returned NULL to keep returning NULL (see
11084 	 * process_iter_next_call()), so remember the drained state.
11085 	 */
11086 	if (!p)
11087 		kit->cursor.flags |= __SCX_DSQ_ITER_DRAINED;
11088 
11089 	return p;
11090 }
11091 
11092 /**
11093  * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator
11094  * @it: iterator to destroy
11095  *
11096  * Undo bpf_iter_scx_dsq_new(). If the iteration marked tasks with
11097  * scx_bpf_dsq_reenq_task(), schedule their re-enqueue.
11098  */
11099 __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
11100 {
11101 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;
11102 
11103 	if (!kit->dsq)
11104 		return;
11105 
11106 	if (!list_empty(&kit->cursor.node)) {
11107 		unsigned long flags;
11108 
11109 		raw_spin_lock_irqsave(&kit->dsq->lock, flags);
11110 		list_del_init(&kit->cursor.node);
11111 		raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
11112 	}
11113 
11114 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_REENQ) {
11115 		guard(preempt)();
11116 		schedule_dsq_reenq(kit->dsq->sched, kit->dsq, SCX_REENQ_MARKED,
11117 				   scx_locked_rq());
11118 	}
11119 
11120 	kit->dsq = NULL;
11121 }
11122 
11123 /**
11124  * scx_bpf_dsq_peek - Lockless peek at the first element.
11125  * @dsq_id: DSQ to examine.
11126  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11127  *
11128  * Read the first element in the DSQ. This is semantically equivalent to using
11129  * the DSQ iterator, but is lockfree. Of course, like any lockless operation,
11130  * this provides only a point-in-time snapshot, and the contents may change
11131  * by the time any subsequent locking operation reads the queue.
11132  *
11133  * Returns the pointer, or NULL indicates an empty queue OR internal error.
11134  */
11135 __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id,
11136 						 const struct bpf_prog_aux *aux)
11137 {
11138 	struct scx_sched *sch;
11139 	struct scx_dispatch_q *dsq;
11140 
11141 	sch = scx_prog_sched(aux);
11142 	if (unlikely(!sch))
11143 		return NULL;
11144 
11145 	if (unlikely(dsq_id & SCX_DSQ_FLAG_BUILTIN)) {
11146 		scx_error(sch, "peek disallowed on builtin DSQ 0x%llx", dsq_id);
11147 		return NULL;
11148 	}
11149 
11150 	dsq = find_user_dsq(sch, dsq_id);
11151 	if (unlikely(!dsq)) {
11152 		scx_error(sch, "peek on non-existent DSQ 0x%llx", dsq_id);
11153 		return NULL;
11154 	}
11155 
11156 	return rcu_dereference(dsq->first_task);
11157 }
11158 
11159 /**
11160  * scx_bpf_dsq_reenq - Re-enqueue tasks on a DSQ
11161  * @dsq_id: DSQ to re-enqueue
11162  * @reenq_flags: %SCX_REENQ_*
11163  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11164  *
11165  * Iterate over all of the tasks currently enqueued on the DSQ identified by
11166  * @dsq_id, and re-enqueue them in the BPF scheduler. The following DSQs are
11167  * supported:
11168  *
11169  * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu)
11170  * - User DSQs
11171  *
11172  * Re-enqueues are performed asynchronously. Can be called from any context
11173  * except NMI. A call from NMI aborts the scheduler. A re-enqueue of an inactive
11174  * cpu's local DSQ is ignored.
11175  *
11176  * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler
11177  * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() -
11178  * or the calling CPU's when no rq is locked.
11179  */
11180 __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,
11181 				   const struct bpf_prog_aux *aux)
11182 {
11183 	struct rq *locked_rq = scx_locked_rq();
11184 	struct scx_sched *sch;
11185 	struct scx_dispatch_q *dsq;
11186 
11187 	guard(preempt)();
11188 
11189 	sch = scx_prog_sched(aux);
11190 	if (unlikely(!sch))
11191 		return;
11192 
11193 	if (unlikely(reenq_flags & ~__SCX_REENQ_USER_MASK)) {
11194 		scx_error(sch, "invalid SCX_REENQ flags 0x%llx", reenq_flags);
11195 		return;
11196 	}
11197 
11198 	if (!scx_kf_allowed_ctx(sch))
11199 		return;
11200 
11201 	/* not specifying any filter bits is the same as %SCX_REENQ_ANY */
11202 	if (!(reenq_flags & __SCX_REENQ_FILTER_MASK))
11203 		reenq_flags |= SCX_REENQ_ANY;
11204 
11205 	dsq = find_dsq_for_dispatch(sch, locked_rq ?: this_rq(), dsq_id, smp_processor_id());
11206 	schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq);
11207 }
11208 
11209 /**
11210  * scx_bpf_dsq_reenq_task - Re-enqueue a task from a DSQ iteration
11211  * @it__iter: DSQ iterator in progress
11212  * @p: task to re-enqueue
11213  *
11214  * Mark @p, which is on the DSQ iterated by @it__iter and was queued before the
11215  * iteration started, for re-enqueue. When the iteration ends, the DSQ's marked
11216  * tasks go through ops.enqueue() with %SCX_ENQ_REENQ in DSQ order and the other
11217  * tasks on the DSQ keep their positions. A marked task that leaves the DSQ in
11218  * the meantime is not re-enqueued. The mark belongs to @p, so a re-enqueue
11219  * scheduled by another iteration of the DSQ may re-enqueue @p before this
11220  * iteration ends. @p must be on the DSQ's scheduler.
11221  *
11222  * Re-enqueues are performed asynchronously. Can be called from ops.dispatch()
11223  * or any BPF context which doesn't hold a rq lock (e.g. BPF timers or SYSCALL
11224  * programs).
11225  *
11226  * Return %true if @p was marked, %false if @p left the DSQ or was queued after
11227  * the iteration started, @p is on another scheduler, or the iterator holds no
11228  * DSQ.
11229  */
11230 __bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter,
11231 					struct task_struct *p)
11232 {
11233 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;
11234 	struct scx_dispatch_q *dsq = kit->dsq;
11235 	struct rq *src_rq, *entry_rq;
11236 	unsigned long flags;
11237 	bool marked = false;
11238 
11239 	/* NULL after a failed bpf_iter_scx_dsq_new(), see scx_dsq_move() */
11240 	if (unlikely(!dsq))
11241 		return false;
11242 
11243 	/* the cursor check needs @p's rq locked */
11244 	src_rq = task_rq(p);
11245 	entry_rq = kf_switch_rq_lock(src_rq, &flags);
11246 
11247 	scoped_guard (raw_spinlock, &dsq->lock) {
11248 		if (nldsq_cursor_lost_task(&kit->cursor, src_rq, dsq, p))
11249 			break;
11250 
11251 		if (unlikely(!scx_task_on_sched(dsq->sched, p))) {
11252 			scx_error(dsq->sched, "scx_bpf_dsq_reenq_task() on %s[%d] but the task belongs to a different scheduler",
11253 				  p->comm, p->pid);
11254 			break;
11255 		}
11256 
11257 		p->scx.dsq_flags |= SCX_TASK_DSQ_REENQ;
11258 		/* the walk is scheduled once, by bpf_iter_scx_dsq_destroy() */
11259 		kit->cursor.flags |= __SCX_DSQ_ITER_HAS_REENQ;
11260 		marked = true;
11261 	}
11262 
11263 	kf_restore_rq_lock(src_rq, entry_rq, flags);
11264 	return marked;
11265 }
11266 
11267 /**
11268  * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ
11269  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11270  *
11271  * Iterate over all of the tasks currently enqueued on the local DSQ of the rq
11272  * the current scheduler operation is locked to, or the calling CPU's when no rq
11273  * is locked, and re-enqueue them in the BPF scheduler. Can be called from any
11274  * context except NMI. A call from NMI aborts the scheduler.
11275  *
11276  * This is now a special case of scx_bpf_dsq_reenq() and may be removed in the
11277  * future.
11278  */
11279 __bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux)
11280 {
11281 	scx_bpf_dsq_reenq(SCX_DSQ_LOCAL, 0, aux);
11282 }
11283 
11284 __bpf_kfunc_end_defs();
11285 
11286 __printf(5, 0)
11287 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,
11288 			 size_t line_size, char *fmt, unsigned long long *data,
11289 			 u32 data__sz)
11290 {
11291 	struct bpf_bprintf_data bprintf_data = { .get_bin_args = true };
11292 	s32 ret;
11293 
11294 	if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 ||
11295 	    (data__sz && !data)) {
11296 		scx_error(sch, "invalid data=%p and data__sz=%u", (void *)data, data__sz);
11297 		return -EINVAL;
11298 	}
11299 
11300 	ret = copy_from_kernel_nofault(data_buf, data, data__sz);
11301 	if (ret < 0) {
11302 		scx_error(sch, "failed to read data fields (%d)", ret);
11303 		return ret;
11304 	}
11305 
11306 	ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8,
11307 				  &bprintf_data);
11308 	if (ret < 0) {
11309 		scx_error(sch, "format preparation failed (%d)", ret);
11310 		return ret;
11311 	}
11312 
11313 	ret = bstr_printf(line_buf, line_size, fmt,
11314 			  bprintf_data.bin_args);
11315 	bpf_bprintf_cleanup(&bprintf_data);
11316 	if (ret < 0) {
11317 		scx_error(sch, "(\"%s\", %p, %u) failed to format", fmt, data, data__sz);
11318 		return ret;
11319 	}
11320 
11321 	return ret;
11322 }
11323 
11324 /*
11325  * Exit @sch with the reason formatted from a BPF-supplied bstr format. The exit
11326  * is claimed first and the reason is formatted directly into the winner-owned
11327  * exit_info buffer, which allows use from any context including NMI.
11328  *
11329  * @fmt_blame is the sched blamed for formatting failures through the
11330  * scx_error() calls in __bstr_format() and differs from @sch when a parent
11331  * supplies the kill reason for a child. A formatting failure doesn't revert the
11332  * claim - @sch still exits with the claimed kind and a fallback message.
11333  */
11334 __printf(5, 0)
11335 bool scx_exit_bstr(struct scx_sched *sch, enum scx_exit_kind kind,
11336 		   s64 exit_code, struct scx_sched *fmt_blame, char *fmt,
11337 		   unsigned long long *data, u32 data__sz)
11338 {
11339 	struct scx_exit_info *ei = sch->exit_info;
11340 	u64 data_buf[MAX_BPRINTF_VARARGS];
11341 	s32 ret;
11342 
11343 	guard(preempt)();
11344 
11345 	if (!scx_claim_exit(sch, kind))
11346 		return false;
11347 
11348 	ret = __bstr_format(fmt_blame, data_buf, ei->msg, SCX_EXIT_MSG_LEN,
11349 			    fmt, data, data__sz);
11350 	if (ret < 0)
11351 		scnprintf(ei->msg, SCX_EXIT_MSG_LEN,
11352 			  "exit message formatting failed (%d)", ret);
11353 
11354 	scx_finish_exit(sch, kind, exit_code, raw_smp_processor_id());
11355 	return true;
11356 }
11357 
11358 __bpf_kfunc_start_defs();
11359 
11360 /**
11361  * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler.
11362  * @exit_code: Exit value to pass to user space via struct scx_exit_info.
11363  * @fmt: error message format string
11364  * @data: format string parameters packaged using ___bpf_fill() macro
11365  * @data__sz: @data len, must end in '__sz' for the verifier
11366  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11367  *
11368  * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops
11369  * disabling.
11370  */
11371 __printf(2, 0)
11372 __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
11373 				   unsigned long long *data, u32 data__sz,
11374 				   const struct bpf_prog_aux *aux)
11375 {
11376 	struct scx_sched *sch;
11377 
11378 	guard(rcu)();
11379 
11380 	sch = scx_prog_sched(aux);
11381 	if (likely(sch))
11382 		scx_exit_bstr(sch, SCX_EXIT_UNREG_BPF, exit_code, sch, fmt,
11383 			      data, data__sz);
11384 }
11385 
11386 /**
11387  * scx_bpf_error_bstr - Indicate fatal error
11388  * @fmt: error message format string
11389  * @data: format string parameters packaged using ___bpf_fill() macro
11390  * @data__sz: @data len, must end in '__sz' for the verifier
11391  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11392  *
11393  * Indicate that the BPF scheduler encountered a fatal error and initiate ops
11394  * disabling.
11395  */
11396 __printf(1, 0)
11397 __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
11398 				    u32 data__sz, const struct bpf_prog_aux *aux)
11399 {
11400 	struct scx_sched *sch;
11401 
11402 	guard(rcu)();
11403 
11404 	sch = scx_prog_sched(aux);
11405 	if (likely(sch))
11406 		scx_exit_bstr(sch, SCX_EXIT_ERROR_BPF, 0, sch, fmt, data,
11407 			      data__sz);
11408 }
11409 
11410 /**
11411  * scx_bpf_dump_bstr - Generate extra debug dump specific to the BPF scheduler
11412  * @fmt: format string
11413  * @data: format string parameters packaged using ___bpf_fill() macro
11414  * @data__sz: @data len, must end in '__sz' for the verifier
11415  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11416  *
11417  * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and
11418  * dump_task() to generate extra debug dump specific to the BPF scheduler.
11419  *
11420  * The extra dump may be multiple lines. A single line may be split over
11421  * multiple calls. The last line is automatically terminated.
11422  */
11423 __printf(1, 0)
11424 __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
11425 				   u32 data__sz, const struct bpf_prog_aux *aux)
11426 {
11427 	struct scx_sched *sch;
11428 	struct scx_dump_data *dd = &scx_dump_data;
11429 	struct scx_bstr_buf *buf = &dd->buf;
11430 	s32 ret;
11431 
11432 	guard(rcu)();
11433 
11434 	sch = scx_prog_sched(aux);
11435 	if (unlikely(!sch))
11436 		return;
11437 
11438 	if (raw_smp_processor_id() != dd->cpu) {
11439 		scx_error(sch, "scx_bpf_dump() must only be called from ops.dump() and friends");
11440 		return;
11441 	}
11442 
11443 	/* append the formatted string to the line buf */
11444 	ret = __bstr_format(sch, buf->data, buf->line + dd->cursor,
11445 			    sizeof(buf->line) - dd->cursor, fmt, data, data__sz);
11446 	if (ret < 0) {
11447 		scx_dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)",
11448 			      dd->prefix, fmt, data, data__sz, ret);
11449 		return;
11450 	}
11451 
11452 	dd->cursor += ret;
11453 	dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line));
11454 
11455 	if (!dd->cursor)
11456 		return;
11457 
11458 	/*
11459 	 * If the line buf overflowed or ends in a newline, flush it into the
11460 	 * dump. This is to allow the caller to generate a single line over
11461 	 * multiple calls. As ops_dump_flush() can also handle multiple lines in
11462 	 * the line buf, the only case which can lead to an unexpected
11463 	 * truncation is when the caller keeps generating newlines in the middle
11464 	 * instead of the end consecutively. Don't do that.
11465 	 */
11466 	if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n')
11467 		ops_dump_flush();
11468 }
11469 
11470 /**
11471  * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU
11472  * @cpu: CPU of interest
11473  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11474  *
11475  * Return the maximum relative capacity of @cpu in relation to the most
11476  * performant CPU in the system. The return value is in the range [1,
11477  * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur().
11478  */
11479 __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux)
11480 {
11481 	struct scx_sched *sch;
11482 
11483 	guard(rcu)();
11484 
11485 	sch = scx_prog_sched(aux);
11486 	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
11487 		return arch_scale_cpu_capacity(cpu);
11488 	else
11489 		return SCX_CPUPERF_ONE;
11490 }
11491 
11492 /**
11493  * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid
11494  * @cid: cid of the CPU to query
11495  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11496  *
11497  * cid-addressed equivalent of scx_bpf_cpuperf_cap().
11498  */
11499 __bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux)
11500 {
11501 	struct scx_sched *sch;
11502 	s32 cpu;
11503 
11504 	guard(rcu)();
11505 
11506 	sch = scx_prog_sched(aux);
11507 	if (unlikely(!sch))
11508 		return SCX_CPUPERF_ONE;
11509 	cpu = scx_cid_to_cpu(sch, cid);
11510 	if (cpu < 0)
11511 		return SCX_CPUPERF_ONE;
11512 	return arch_scale_cpu_capacity(cpu);
11513 }
11514 
11515 /**
11516  * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU
11517  * @cpu: CPU of interest
11518  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11519  *
11520  * Return the current relative performance of @cpu in relation to its maximum.
11521  * The return value is in the range [1, %SCX_CPUPERF_ONE].
11522  *
11523  * The current performance level of a CPU in relation to the maximum performance
11524  * available in the system can be calculated as follows:
11525  *
11526  *   scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE
11527  *
11528  * The result is in the range [1, %SCX_CPUPERF_ONE].
11529  */
11530 __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux)
11531 {
11532 	struct scx_sched *sch;
11533 
11534 	guard(rcu)();
11535 
11536 	sch = scx_prog_sched(aux);
11537 	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
11538 		return arch_scale_freq_capacity(cpu);
11539 	else
11540 		return SCX_CPUPERF_ONE;
11541 }
11542 
11543 /**
11544  * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid
11545  * @cid: cid of the CPU to query
11546  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11547  *
11548  * cid-addressed equivalent of scx_bpf_cpuperf_cur().
11549  */
11550 __bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux)
11551 {
11552 	struct scx_sched *sch;
11553 	s32 cpu;
11554 
11555 	guard(rcu)();
11556 
11557 	sch = scx_prog_sched(aux);
11558 	if (unlikely(!sch))
11559 		return SCX_CPUPERF_ONE;
11560 	cpu = scx_cid_to_cpu(sch, cid);
11561 	if (cpu < 0)
11562 		return SCX_CPUPERF_ONE;
11563 	return arch_scale_freq_capacity(cpu);
11564 }
11565 
11566 /* validate and apply a cpuperf target, see scx_bpf_cpuperf_set() */
11567 static s32 scx_cpuperf_set(struct scx_sched *sch, s32 cpu, u32 perf)
11568 {
11569 	struct rq *rq, *locked_rq;
11570 	struct rq_flags rf;
11571 	s32 ret;
11572 
11573 	if (unlikely(perf > SCX_CPUPERF_ONE)) {
11574 		scx_error(sch, "Invalid cpuperf target %u for CPU %d", perf, cpu);
11575 		return -EINVAL;
11576 	}
11577 
11578 	if (!scx_cpu_valid(sch, cpu, NULL))
11579 		return -EINVAL;
11580 
11581 	rq = cpu_rq(cpu);
11582 	locked_rq = scx_locked_rq();
11583 
11584 	/*
11585 	 * When called with an rq lock held, restrict the operation to the
11586 	 * corresponding CPU to prevent ABBA deadlocks.
11587 	 */
11588 	if (locked_rq && rq != locked_rq) {
11589 		scx_error(sch, "Invalid target CPU %d", cpu);
11590 		return -EINVAL;
11591 	}
11592 
11593 	/*
11594 	 * If no rq lock is held, allow to operate on any CPU by acquiring
11595 	 * the corresponding rq lock.
11596 	 */
11597 	if (!locked_rq) {
11598 		rq_lock_irqsave(rq, &rf);
11599 		update_rq_clock(rq);
11600 	}
11601 
11602 	/*
11603 	 * ecaps updates are folded under the rq lock, making this test
11604 	 * authoritative: a write can never land after a revoke has taken
11605 	 * effect on @cpu.
11606 	 */
11607 	if (likely(!scx_missing_caps(sch, cpu, SCX_CAP_PERF))) {
11608 		rq->scx.cpuperf_target = perf;
11609 		cpufreq_update_util(rq, 0);
11610 		ret = 0;
11611 	} else {
11612 		__scx_add_event(sch, SCX_EV_SUB_CIDPERF_DENIED, 1);
11613 		ret = -EACCES;
11614 	}
11615 
11616 	if (!locked_rq)
11617 		rq_unlock_irqrestore(rq, &rf);
11618 
11619 	return ret;
11620 }
11621 
11622 /**
11623  * scx_bpf_cpuperf_set - Set the relative performance target of a CPU
11624  * @cpu: CPU of interest
11625  * @perf: target performance level [0, %SCX_CPUPERF_ONE]
11626  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11627  *
11628  * Set the target performance level of @cpu to @perf. @perf is in linear
11629  * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the
11630  * schedutil cpufreq governor chooses the target frequency.
11631  *
11632  * The actual performance level chosen, CPU grouping, and the overhead and
11633  * latency of the operations are dependent on the hardware and cpufreq driver in
11634  * use. Consult hardware and cpufreq documentation for more information. The
11635  * current performance level can be monitored using scx_bpf_cpuperf_cur().
11636  */
11637 __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_aux *aux)
11638 {
11639 	struct scx_sched *sch;
11640 
11641 	guard(rcu)();
11642 
11643 	sch = scx_prog_sched(aux);
11644 	if (unlikely(!sch))
11645 		return;
11646 
11647 	if (!scx_kf_allowed_ctx(sch))
11648 		return;
11649 
11650 	scx_cpuperf_set(sch, cpu, perf);
11651 }
11652 
11653 /**
11654  * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid
11655  * @cid: cid of the CPU to target
11656  * @perf: target performance level [0, %SCX_CPUPERF_ONE]
11657  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11658  *
11659  * cid-addressed equivalent of scx_bpf_cpuperf_set(). A sub-sched needs
11660  * SCX_CAP_PERF on @cid. Returns 0 if the target was applied, -%EACCES if
11661  * the write was denied for missing caps, other -errnos if @cid didn't
11662  * resolve.
11663  */
11664 __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf,
11665 				    const struct bpf_prog_aux *aux)
11666 {
11667 	struct scx_sched *sch;
11668 	s32 cpu;
11669 
11670 	guard(rcu)();
11671 
11672 	sch = scx_prog_sched(aux);
11673 	if (unlikely(!sch))
11674 		return -ENODEV;
11675 
11676 	if (!scx_kf_allowed_ctx(sch))
11677 		return -EDEADLK;
11678 
11679 	cpu = scx_cid_to_cpu(sch, cid);
11680 	if (cpu < 0)
11681 		return cpu;
11682 
11683 	return scx_cpuperf_set(sch, cpu, perf);
11684 }
11685 
11686 /**
11687  * scx_bpf_nr_node_ids - Return the number of possible node IDs
11688  *
11689  * All valid node IDs in the system are smaller than the returned value.
11690  */
11691 __bpf_kfunc u32 scx_bpf_nr_node_ids(void)
11692 {
11693 	return nr_node_ids;
11694 }
11695 
11696 /**
11697  * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs
11698  *
11699  * All valid CPU IDs in the system are smaller than the returned value.
11700  */
11701 __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void)
11702 {
11703 	return nr_cpu_ids;
11704 }
11705 
11706 /**
11707  * scx_bpf_nr_cids - Return the size of the cid space
11708  *
11709  * Equals num_possible_cpus(). All valid cids are in [0, return value).
11710  */
11711 __bpf_kfunc u32 scx_bpf_nr_cids(void)
11712 {
11713 	return num_possible_cpus();
11714 }
11715 
11716 /**
11717  * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space
11718  *
11719  * Return num_online_cpus(). The standard model restarts the scheduler on
11720  * hotplug, which lets schedulers treat [0, nr_online_cids) as the online
11721  * range. Schedulers that prefer to handle hotplug without a restart should
11722  * install a custom mapping via scx_bpf_cid_override() and track onlining
11723  * through the ops.cid_online / ops.cid_offline callbacks, starting from the
11724  * mask scx_bpf_online_cmask() returns.
11725  */
11726 __bpf_kfunc u32 scx_bpf_nr_online_cids(void)
11727 {
11728 	return num_online_cpus();
11729 }
11730 
11731 /**
11732  * scx_bpf_online_cmask - Return the online cid mask in the scheduler arena
11733  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11734  *
11735  * Return a kernel-maintained cmask covering [0, scx_bpf_nr_cids()), or NULL if
11736  * the calling program is not associated with a live cid-form scheduler or the
11737  * mask is not allocated yet, as in ops.init_cids(). Treat the mask as read-only
11738  * even though arena memory stays writable by the BPF scheduler. The mask
11739  * follows the SCX hotplug notifications: a cid's bit is updated before
11740  * ops.cid_online/offline() runs for it. The pointer is valid from ops.init()
11741  * through ops.exit(). Root ops.init() runs with hotplug excluded. Other
11742  * contexts can observe concurrent updates.
11743  */
11744 __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux)
11745 {
11746 	struct scx_sched *sch;
11747 	struct scx_cmask *online;
11748 
11749 	guard(rcu)();
11750 
11751 	sch = scx_prog_sched(aux);
11752 	if (unlikely(!sch))
11753 		return NULL;
11754 	online = sch->online_cmask;
11755 	if (unlikely(!online))
11756 		return NULL;
11757 
11758 	return (void *)scx_kaddr_to_arena(sch, online);
11759 }
11760 
11761 /**
11762  * scx_bpf_this_cid - Return the cid of the CPU this program is running on
11763  *
11764  * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs.
11765  * The current cpu is trivially valid, so this is just a table lookup. Return
11766  * -EINVAL if called before any scheduler has ever published its cid tables.
11767  */
11768 __bpf_kfunc s32 scx_bpf_this_cid(void)
11769 {
11770 	s16 *tbl;
11771 
11772 	guard(rcu)();
11773 
11774 	tbl = rcu_dereference(scx_cpu_to_cid_tbl);
11775 	if (!tbl)
11776 		return -EINVAL;
11777 	return tbl[raw_smp_processor_id()];
11778 }
11779 
11780 /**
11781  * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask
11782  */
11783 __bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void)
11784 {
11785 	return cpu_possible_mask;
11786 }
11787 
11788 /**
11789  * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask
11790  */
11791 __bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void)
11792 {
11793 	return cpu_online_mask;
11794 }
11795 
11796 /**
11797  * scx_bpf_put_cpumask - Release a possible/online cpumask
11798  * @cpumask: cpumask to release
11799  */
11800 __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask)
11801 {
11802 	/*
11803 	 * Empty function body because we aren't actually acquiring or releasing
11804 	 * a reference to a global cpumask, which is read-only in the caller and
11805 	 * is never released. The acquire / release semantics here are just used
11806 	 * to make the cpumask is a trusted pointer in the caller.
11807 	 */
11808 }
11809 
11810 /**
11811  * scx_bpf_task_running - Is task the current scheduling context?
11812  * @p: task of interest
11813  *
11814  * Under proxy execution, this reports the donor rather than the task whose
11815  * code is physically executing. The physical execution context is intentionally
11816  * not exposed to the BPF scheduler, which continues to observe the donor as the
11817  * running scheduling context.
11818  */
11819 __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p)
11820 {
11821 	return rcu_access_pointer(task_rq(p)->donor) == p;
11822 }
11823 
11824 /**
11825  * scx_bpf_task_cpu - CPU a task is currently associated with
11826  * @p: task of interest
11827  */
11828 __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p)
11829 {
11830 	return task_cpu(p);
11831 }
11832 
11833 /**
11834  * scx_bpf_task_cid - cid a task is currently associated with
11835  * @p: task of interest
11836  *
11837  * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a
11838  * valid cpu, so this is just a table lookup. Return -EINVAL if called before
11839  * any scheduler has ever published its cid tables.
11840  */
11841 __bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p)
11842 {
11843 	s16 *tbl;
11844 
11845 	/* KF_RCU covers only @p - a sleepable program holds no RCU lock */
11846 	guard(rcu)();
11847 
11848 	tbl = rcu_dereference(scx_cpu_to_cid_tbl);
11849 	if (!tbl)
11850 		return -EINVAL;
11851 	return tbl[task_cpu(p)];
11852 }
11853 
11854 /**
11855  * scx_bpf_locked_rq - Return the rq currently locked by SCX
11856  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11857  *
11858  * Returns the rq if a rq lock is currently held by SCX.
11859  * Otherwise emits an error and returns NULL.
11860  */
11861 __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux)
11862 {
11863 	struct scx_sched *sch;
11864 	struct rq *rq;
11865 
11866 	guard(preempt)();
11867 
11868 	sch = scx_prog_sched(aux);
11869 	if (unlikely(!sch))
11870 		return NULL;
11871 
11872 	rq = scx_locked_rq();
11873 	if (!rq) {
11874 		scx_error(sch, "accessing rq without holding rq lock");
11875 		return NULL;
11876 	}
11877 
11878 	return rq;
11879 }
11880 
11881 /**
11882  * scx_bpf_cpu_curr - Return remote CPU's current scheduling context
11883  * @cpu: CPU of interest
11884  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11885  *
11886  * Under proxy execution, this returns the donor, which supplies the scheduling
11887  * policy and runtime budget, rather than the task whose code is physically
11888  * executing. The physical execution context is intentionally not exposed to
11889  * the BPF scheduler.
11890  *
11891  * Callers must hold RCU read lock (KF_RCU).
11892  */
11893 __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux)
11894 {
11895 	struct scx_sched *sch;
11896 
11897 	guard(rcu)();
11898 
11899 	sch = scx_prog_sched(aux);
11900 	if (unlikely(!sch))
11901 		return NULL;
11902 
11903 	if (!scx_cpu_valid(sch, cpu, NULL))
11904 		return NULL;
11905 
11906 	return rcu_dereference(cpu_rq(cpu)->donor);
11907 }
11908 
11909 /**
11910  * scx_bpf_cid_curr - Return the curr task on the CPU at @cid
11911  * @cid: cid of interest
11912  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
11913  *
11914  * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU
11915  * read lock (KF_RCU).
11916  */
11917 __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux)
11918 {
11919 	struct scx_sched *sch;
11920 	s32 cpu;
11921 
11922 	guard(rcu)();
11923 
11924 	sch = scx_prog_sched(aux);
11925 	if (unlikely(!sch))
11926 		return NULL;
11927 	cpu = scx_cid_to_cpu(sch, cid);
11928 	if (cpu < 0)
11929 		return NULL;
11930 	return rcu_dereference(cpu_rq(cpu)->donor);
11931 }
11932 
11933 /**
11934  * scx_bpf_tid_to_task - Look up a task by its scx tid
11935  * @tid: task ID previously read from p->scx.tid
11936  *
11937  * Returns the task with the given tid, or NULL if no such task exists. The
11938  * returned pointer is valid until the end of the current RCU read section
11939  * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root
11940  * scheduler; otherwise an error is raised and NULL returned.
11941  */
11942 __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid)
11943 {
11944 	struct sched_ext_entity *scx;
11945 
11946 	if (!scx_tid_to_task_enabled()) {
11947 		struct scx_sched *sch = rcu_dereference(scx_root);
11948 
11949 		if (sch)
11950 			scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK");
11951 		return NULL;
11952 	}
11953 
11954 	scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params);
11955 	if (!scx)
11956 		return NULL;
11957 
11958 	return container_of(scx, struct task_struct, scx);
11959 }
11960 
11961 u64 __scx_bpf_now(struct rq *rq)
11962 {
11963 	/* the caller must be on @rq's cpu or hold its lock */
11964 	lockdep_assert((rq == this_rq() && !preemptible()) ||
11965 		       lockdep_is_held(__rq_lockp(rq)));
11966 
11967 	if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) {
11968 		/* if the rq clock is valid, use the cached rq clock */
11969 		return READ_ONCE(rq->scx.clock);
11970 	} else {
11971 		/*
11972 		 * Otherwise, return a fresh rq clock.
11973 		 *
11974 		 * The rq clock is updated outside of the rq lock.
11975 		 * In this case, keep the updated rq clock invalid so the next
11976 		 * read outside the rq lock gets a fresh rq clock.
11977 		 */
11978 		return sched_clock_cpu(cpu_of(rq));
11979 	}
11980 }
11981 
11982 /**
11983  * scx_bpf_now - Returns a high-performance monotonically non-decreasing
11984  * clock for the current CPU. The clock returned is in nanoseconds.
11985  *
11986  * It provides the following properties:
11987  *
11988  * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently
11989  *  to account for execution time and track tasks' runtime properties.
11990  *  Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which
11991  *  eventually reads a hardware timestamp counter -- is neither performant nor
11992  *  scalable. scx_bpf_now() aims to provide a high-performance clock by
11993  *  using the rq clock in the scheduler core whenever possible.
11994  *
11995  * 2) High enough resolution for the BPF scheduler use cases: In most BPF
11996  *  scheduler use cases, the required clock resolution is lower than the most
11997  *  accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically
11998  *  uses the rq clock in the scheduler core whenever it is valid. It considers
11999  *  that the rq clock is valid from the time the rq clock is updated
12000  *  (update_rq_clock) until the rq is unlocked (rq_unpin_lock).
12001  *
12002  * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now()
12003  *  guarantees the clock never goes backward when comparing them in the same
12004  *  CPU. On the other hand, when comparing clocks in different CPUs, there
12005  *  is no such guarantee -- the clock can go backward. It provides a
12006  *  monotonically *non-decreasing* clock so that it would provide the same
12007  *  clock values in two different scx_bpf_now() calls in the same CPU
12008  *  during the same period of when the rq clock is valid.
12009  */
12010 __bpf_kfunc u64 scx_bpf_now(void)
12011 {
12012 	/*
12013 	 * Note that scx_bpf_now() is re-entrant between a process context and
12014 	 * an interrupt context (e.g., timer interrupt). However, we don't need
12015 	 * to consider the race between them because such race is not observable
12016 	 * from a caller.
12017 	 */
12018 	guard(preempt)();
12019 	return __scx_bpf_now(this_rq());
12020 }
12021 
12022 static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events)
12023 {
12024 	int cpu;
12025 
12026 	/* Aggregate per-CPU event counters into @events. */
12027 	memset(events, 0, sizeof(*events));
12028 	for_each_possible_cpu(cpu) {
12029 		struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats;
12030 #define SCX_EVENT(name)	(events->name += READ_ONCE(e_cpu->name))
12031 		SCX_EVENTS_LIST(SCX_EVENT);
12032 #undef SCX_EVENT
12033 	}
12034 }
12035 
12036 /**
12037  * scx_bpf_events - Read the event counters of the calling scheduler
12038  * @events: output buffer from a BPF program
12039  * @events__sz: @events len, must end in '__sz' for the verifier
12040  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
12041  *
12042  * Read the event counters of the scheduler associated with the calling program.
12043  * @events is zeroed when no scheduler can be resolved.
12044  */
12045 __bpf_kfunc void scx_bpf_events(struct scx_event_stats *events, size_t events__sz,
12046 				const struct bpf_prog_aux *aux)
12047 {
12048 	struct scx_sched *sch;
12049 	struct scx_event_stats e_sys;
12050 
12051 	rcu_read_lock();
12052 	sch = scx_prog_sched(aux);
12053 	if (sch)
12054 		scx_read_events(sch, &e_sys);
12055 	else
12056 		memset(&e_sys, 0, sizeof(e_sys));
12057 	rcu_read_unlock();
12058 
12059 	/*
12060 	 * We cannot entirely trust a BPF-provided size since a BPF program
12061 	 * might be compiled against a different vmlinux.h, of which
12062 	 * scx_event_stats would be larger (a newer vmlinux.h) or smaller
12063 	 * (an older vmlinux.h). Hence, we use the smaller size to avoid
12064 	 * memory corruption.
12065 	 */
12066 	events__sz = min(events__sz, sizeof(*events));
12067 	memcpy(events, &e_sys, events__sz);
12068 }
12069 
12070 #ifdef CONFIG_CGROUP_SCHED
12071 /**
12072  * scx_bpf_task_cgroup - Return the sched cgroup of a task
12073  * @p: task of interest
12074  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
12075  *
12076  * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with
12077  * from the scheduler's POV. SCX operations should use this function to
12078  * determine @p's current cgroup as, unlike following @p->cgroups,
12079  * @p->sched_task_group is stable for the duration of the SCX op. See
12080  * SCX_CALL_OP_TASK() for details.
12081  */
12082 __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p,
12083 					       const struct bpf_prog_aux *aux)
12084 {
12085 	struct task_group *tg = p->sched_task_group;
12086 	struct cgroup *cgrp = &cgrp_dfl_root.cgrp;
12087 	struct scx_sched *sch;
12088 
12089 	guard(rcu)();
12090 
12091 	sch = scx_prog_sched(aux);
12092 	if (unlikely(!sch))
12093 		goto out;
12094 
12095 	if (!scx_kf_arg_task_ok(sch, p))
12096 		goto out;
12097 
12098 	cgrp = tg_cgrp(tg);
12099 
12100 out:
12101 	cgroup_get(cgrp);
12102 	return cgrp;
12103 }
12104 
12105 /**
12106  * scx_bpf_cgroup_nr_cpus - Return the number of CPUs in a cgroup's cpuset
12107  * @cgrp: cgroup of interest
12108  *
12109  * Return the number of CPUs in @cgrp's effective cpuset, which is inherited
12110  * from the nearest ancestor with the cpuset controller enabled. This matches
12111  * the count used by fair's group share calculation and lets hierarchical BPF
12112  * schedulers bound a group's weight by the CPUs it can actually run on.
12113  *
12114  * The value is a snapshot and may change at any time through cpuset updates or
12115  * CPU hotplug. Schedulers should re-read it when recomputing group state.
12116  */
12117 __bpf_kfunc u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp)
12118 {
12119 	return cpuset_num_cpus(cgrp);
12120 }
12121 #endif	/* CONFIG_CGROUP_SCHED */
12122 
12123 __bpf_kfunc_end_defs();
12124 
12125 BTF_KFUNCS_START(scx_kfunc_ids_any)
12126 BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU);
12127 BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU);
12128 BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU);
12129 BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
12130 BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS)
12131 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
12132 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
12133 BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL)
12134 BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)
12135 BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)
12136 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_IMPLICIT_ARGS | KF_ITER_NEW | KF_RCU_PROTECTED)
12137 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
12138 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
12139 BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_IMPLICIT_ARGS)
12140 BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_IMPLICIT_ARGS)
12141 BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS)
12142 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
12143 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)
12144 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
12145 BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS)
12146 BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS)
12147 BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS)
12148 BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
12149 BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
12150 BTF_ID_FLAGS(func, scx_bpf_nr_cids)
12151 BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)
12152 BTF_ID_FLAGS(func, scx_bpf_online_cmask, KF_IMPLICIT_ARGS | KF_ARENA_RET)
12153 BTF_ID_FLAGS(func, scx_bpf_this_cid)
12154 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
12155 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
12156 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
12157 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)
12158 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
12159 BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU)
12160 BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL)
12161 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
12162 BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
12163 BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED)
12164 BTF_ID_FLAGS(func, scx_bpf_now)
12165 BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS)
12166 #ifdef CONFIG_CGROUP_SCHED
12167 BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE)
12168 BTF_ID_FLAGS(func, scx_bpf_cgroup_nr_cpus, KF_RCU)
12169 #endif
12170 BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS)
12171 BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS)
12172 BTF_ID_FLAGS(func, scx_bpf_sub_caps, KF_IMPLICIT_ARGS)
12173 BTF_ID_FLAGS(func, scx_bpf_sub_kill_bstr, KF_IMPLICIT_ARGS)
12174 BTF_KFUNCS_END(scx_kfunc_ids_any)
12175 
12176 static const struct btf_kfunc_id_set scx_kfunc_set_any = {
12177 	.owner			= THIS_MODULE,
12178 	.set			= &scx_kfunc_ids_any,
12179 	.filter			= scx_kfunc_context_filter,
12180 };
12181 
12182 /*
12183  * Kfuncs that tracing programs can't call. A tracing program can run in any
12184  * kernel context, including with an rq lock held. scx_locked_rq() only tracks
12185  * rq locks held across SCX callbacks and returns NULL there.
12186  *
12187  * scx_bpf_cpuperf_set() and scx_bpf_cidperf_set() take the target cpu's rq lock
12188  * when scx_locked_rq() returns NULL. Taking it while the context already holds
12189  * an rq lock, the same one or another cpu's, can deadlock.
12190  *
12191  * scx_bpf_destroy_dsq() takes a user DSQ lock. So do
12192  * bpf_iter_scx_dsq_next() and bpf_iter_scx_dsq_destroy(), the latter only
12193  * after a successful next() has linked the cursor. A tracing program can
12194  * run while that lock is already held and recursively acquire it.
12195  *
12196  * Before returning, scx_bpf_sub_grant() and scx_bpf_sub_revoke() invoke
12197  * ops.sub_caps_updated() of the sub-schedulers whose caps changed, so those ops
12198  * run in the tracing program's context. No SCX op expects to run from an
12199  * arbitrary tracing context. One that calls scx_bpf_cidperf_set() hits the
12200  * deadlock above.
12201  *
12202  * Members must also stay in scx_kfunc_ids_any so that struct_ops and syscall
12203  * programs keep their access.
12204  */
12205 BTF_KFUNCS_START(scx_kfunc_ids_no_tracing)
12206 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
12207 BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS)
12208 BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS)
12209 BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS)
12210 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
12211 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
12212 BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
12213 BTF_KFUNCS_END(scx_kfunc_ids_no_tracing)
12214 
12215 /*
12216  * cpu-form kfuncs that are forbidden from cid-form schedulers
12217  * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must
12218  * use the cid-form alternative (cid/cmask kfuncs).
12219  *
12220  * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter
12221  * tests this set independently and rejects matches before the per-op
12222  * allow-list check runs.
12223  *
12224  * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and
12225  * intersects flags across duplicate entries, so each entry must carry the
12226  * same flags as the kfunc's primary declaration; otherwise the flags get
12227  * dropped globally.
12228  */
12229 BTF_KFUNCS_START(scx_kfunc_ids_cpu_only)
12230 BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
12231 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
12232 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
12233 BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS)
12234 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
12235 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)
12236 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
12237 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
12238 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
12239 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
12240 BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU)
12241 BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU)
12242 BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)
12243 BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE)
12244 BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)
12245 BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE)
12246 BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)
12247 BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)
12248 BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS)
12249 BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU)
12250 BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)
12251 BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU)
12252 BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)
12253 BTF_KFUNCS_END(scx_kfunc_ids_cpu_only)
12254 
12255 /*
12256  * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc
12257  * group; an op may permit zero or more groups, with the union expressed in
12258  * scx_kf_allow_flags[]. The verifier-time filter (scx_kfunc_context_filter())
12259  * consults this table to decide whether a context-sensitive kfunc is callable
12260  * from a given SCX op.
12261  */
12262 enum scx_kf_allow_flags {
12263 	SCX_KF_ALLOW_UNLOCKED		= 1 << 0,
12264 	SCX_KF_ALLOW_INIT_CIDS		= 1 << 1,
12265 	SCX_KF_ALLOW_CPU_RELEASE	= 1 << 2,
12266 	SCX_KF_ALLOW_DISPATCH		= 1 << 3,
12267 	SCX_KF_ALLOW_ENQUEUE		= 1 << 4,
12268 	SCX_KF_ALLOW_SELECT_CPU		= 1 << 5,
12269 };
12270 
12271 /*
12272  * Map each SCX op to the union of kfunc groups it permits, indexed by
12273  * SCX_OP_IDX(op). Ops not listed only permit kfuncs that are not
12274  * context-sensitive.
12275  */
12276 static const u32 scx_kf_allow_flags[] = {
12277 	[SCX_OP_IDX(select_cpu)]	= SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE,
12278 	[SCX_OP_IDX(enqueue)]		= SCX_KF_ALLOW_SELECT_CPU | SCX_KF_ALLOW_ENQUEUE,
12279 	[SCX_OP_IDX(dispatch)]		= SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH,
12280 	[SCX_OP_IDX(cpu_release)]	= SCX_KF_ALLOW_CPU_RELEASE,
12281 	[SCX_OP_IDX(init_task)]		= SCX_KF_ALLOW_UNLOCKED,
12282 	[SCX_OP_IDX(dump)]		= SCX_KF_ALLOW_UNLOCKED,
12283 #ifdef CONFIG_EXT_GROUP_SCHED
12284 	[SCX_OP_IDX(cgroup_init)]	= SCX_KF_ALLOW_UNLOCKED,
12285 	[SCX_OP_IDX(cgroup_exit)]	= SCX_KF_ALLOW_UNLOCKED,
12286 	[SCX_OP_IDX(cgroup_prep_move)]	= SCX_KF_ALLOW_UNLOCKED,
12287 	[SCX_OP_IDX(cgroup_cancel_move)] = SCX_KF_ALLOW_UNLOCKED,
12288 	[SCX_OP_IDX(cgroup_set_weight)]	= SCX_KF_ALLOW_UNLOCKED,
12289 	[SCX_OP_IDX(cgroup_set_bandwidth)] = SCX_KF_ALLOW_UNLOCKED,
12290 	[SCX_OP_IDX(cgroup_set_idle)]	= SCX_KF_ALLOW_UNLOCKED,
12291 #endif	/* CONFIG_EXT_GROUP_SCHED */
12292 	[SCX_OP_IDX(sub_attach)]	= SCX_KF_ALLOW_UNLOCKED,
12293 	[SCX_OP_IDX(sub_detach)]	= SCX_KF_ALLOW_UNLOCKED,
12294 	[SCX_OP_IDX(sub_ecaps_updated)]	= SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH,
12295 	[SCX_OP_IDX(sub_child_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH,
12296 	[SCX_OP_IDX(cpu_online)]	= SCX_KF_ALLOW_UNLOCKED,
12297 	[SCX_OP_IDX(cpu_offline)]	= SCX_KF_ALLOW_UNLOCKED,
12298 	[SCX_OP_IDX(init_cids)]		= SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS,
12299 	[SCX_OP_IDX(init)]		= SCX_KF_ALLOW_UNLOCKED,
12300 	[SCX_OP_IDX(exit)]		= SCX_KF_ALLOW_UNLOCKED,
12301 };
12302 
12303 /*
12304  * Verifier-time filter for SCX kfuncs. Registered via the .filter field on
12305  * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc
12306  * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or
12307  * BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the
12308  * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by
12309  * falling through to "allow" when none of the SCX sets contain the kfunc.
12310  */
12311 int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
12312 {
12313 	bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id);
12314 	bool in_init_cids = btf_id_set8_contains(&scx_kfunc_ids_init_cids, kfunc_id);
12315 	bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id);
12316 	bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);
12317 	bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);
12318 	bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id);
12319 	bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id);
12320 	bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id);
12321 	bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id);
12322 	bool in_cid = btf_id_set8_contains(&scx_kfunc_ids_cid, kfunc_id);
12323 	bool in_no_tracing = btf_id_set8_contains(&scx_kfunc_ids_no_tracing, kfunc_id);
12324 	u32 moff, flags;
12325 
12326 	/* Not an SCX kfunc - allow. */
12327 	if (!(in_unlocked || in_init_cids || in_select_cpu || in_enqueue || in_dispatch ||
12328 	      in_cpu_release || in_idle || in_any || in_cid))
12329 		return 0;
12330 
12331 	/* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */
12332 	if (prog->type == BPF_PROG_TYPE_SYSCALL)
12333 		return (in_unlocked || in_select_cpu || in_idle || in_any || in_cid) ? 0 : -EACCES;
12334 
12335 	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
12336 		return (!in_no_tracing && (in_any || in_idle || in_cid)) ? 0 : -EACCES;
12337 
12338 	/*
12339 	 * add_subprog_and_kfunc() collects all kfunc calls, including dead code
12340 	 * guarded by bpf_ksym_exists(), before check_attach_btf_id() sets
12341 	 * prog->aux->st_ops. Allow all kfuncs when st_ops is not yet set;
12342 	 * do_check_main() re-runs the filter with st_ops set and enforces the
12343 	 * actual restrictions.
12344 	 */
12345 	if (!prog->aux->st_ops)
12346 		return 0;
12347 
12348 	/*
12349 	 * Non-SCX struct_ops: SCX kfuncs are not permitted.
12350 	 *
12351 	 * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid
12352 	 * (cid-form) are valid SCX struct_ops. Member offsets match between
12353 	 * the two (verified by BUILD_BUG_ON in scx_init()), so the shared
12354 	 * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to
12355 	 * both.
12356 	 */
12357 	if (prog->aux->st_ops != &bpf_sched_ext_ops &&
12358 	    prog->aux->st_ops != &bpf_sched_ext_ops_cid)
12359 		return -EACCES;
12360 
12361 	/*
12362 	 * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both
12363 	 * small s32s and trivially confused, so cpu-only kfuncs are rejected at
12364 	 * load time. The reverse (cpu-form calling cid-form kfuncs) is
12365 	 * intentionally permissive to ease gradual cpumask -> cid migration.
12366 	 */
12367 	if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only)
12368 		return -EACCES;
12369 
12370 	/* SCX struct_ops: check the per-op allow list. */
12371 	if (in_any || in_idle || in_cid)
12372 		return 0;
12373 
12374 	moff = prog->aux->attach_st_ops_member_off;
12375 	flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)];
12376 
12377 	if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked)
12378 		return 0;
12379 	if ((flags & SCX_KF_ALLOW_INIT_CIDS) && in_init_cids)
12380 		return 0;
12381 	if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release)
12382 		return 0;
12383 	if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch)
12384 		return 0;
12385 	if ((flags & SCX_KF_ALLOW_ENQUEUE) && in_enqueue)
12386 		return 0;
12387 	if ((flags & SCX_KF_ALLOW_SELECT_CPU) && in_select_cpu)
12388 		return 0;
12389 
12390 	return -EACCES;
12391 }
12392 
12393 static int __init scx_init(void)
12394 {
12395 	int ret;
12396 
12397 	/*
12398 	 * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv.
12399 	 * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets
12400 	 * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets
12401 	 * matching for the shared fields. Catch any drift at boot.
12402 	 */
12403 #define CID_OFFSET_MATCH(cpu_field, cid_field)					\
12404 	BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) !=		\
12405 		     offsetof(struct sched_ext_ops_cid, cid_field))
12406 	/* data fields used by bpf_scx_init_member() */
12407 	CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch);
12408 	CID_OFFSET_MATCH(flags, flags);
12409 	CID_OFFSET_MATCH(name, name);
12410 	CID_OFFSET_MATCH(timeout_ms, timeout_ms);
12411 	CID_OFFSET_MATCH(exit_dump_len, exit_dump_len);
12412 	CID_OFFSET_MATCH(hotplug_seq, hotplug_seq);
12413 	CID_OFFSET_MATCH(cid_shard_size, cid_shard_size);
12414 	CID_OFFSET_MATCH(rescue_bandwidth_ppt, rescue_bandwidth_ppt);
12415 	CID_OFFSET_MATCH(rescue_quantum_us, rescue_quantum_us);
12416 	CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id);
12417 	/* shared callbacks: the union view requires byte-for-byte offset match */
12418 	CID_OFFSET_MATCH(enqueue, enqueue);
12419 	CID_OFFSET_MATCH(dequeue, dequeue);
12420 	CID_OFFSET_MATCH(dispatch, dispatch);
12421 	CID_OFFSET_MATCH(tick, tick);
12422 	CID_OFFSET_MATCH(runnable, runnable);
12423 	CID_OFFSET_MATCH(running, running);
12424 	CID_OFFSET_MATCH(stopping, stopping);
12425 	CID_OFFSET_MATCH(quiescent, quiescent);
12426 	CID_OFFSET_MATCH(yield, yield);
12427 	CID_OFFSET_MATCH(core_sched_before, core_sched_before);
12428 	CID_OFFSET_MATCH(set_weight, set_weight);
12429 	CID_OFFSET_MATCH(update_idle, update_idle);
12430 	CID_OFFSET_MATCH(init_task, init_task);
12431 	CID_OFFSET_MATCH(exit_task, exit_task);
12432 	CID_OFFSET_MATCH(enable, enable);
12433 	CID_OFFSET_MATCH(disable, disable);
12434 	CID_OFFSET_MATCH(dump, dump);
12435 	CID_OFFSET_MATCH(dump_task, dump_task);
12436 	CID_OFFSET_MATCH(sub_attach, sub_attach);
12437 	CID_OFFSET_MATCH(sub_detach, sub_detach);
12438 	CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated);
12439 	CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated);
12440 	CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated);
12441 	CID_OFFSET_MATCH(sub_child_ecaps_updated, sub_child_ecaps_updated);
12442 	CID_OFFSET_MATCH(init_cids, init_cids);
12443 	CID_OFFSET_MATCH(init, init);
12444 	CID_OFFSET_MATCH(exit, exit);
12445 	/* renamed callbacks must occupy the same slot as their cpu-form sibling */
12446 	CID_OFFSET_MATCH(select_cpu, select_cid);
12447 	CID_OFFSET_MATCH(set_cpumask, set_cmask);
12448 	CID_OFFSET_MATCH(cpu_online, cid_online);
12449 	CID_OFFSET_MATCH(cpu_offline, cid_offline);
12450 	CID_OFFSET_MATCH(dump_cpu, dump_cid);
12451 #ifdef CONFIG_EXT_GROUP_SCHED
12452 	CID_OFFSET_MATCH(cgroup_init, cpuctl_init);
12453 	CID_OFFSET_MATCH(cgroup_exit, cpuctl_exit);
12454 	CID_OFFSET_MATCH(cgroup_prep_move, cpuctl_prep_move);
12455 	CID_OFFSET_MATCH(cgroup_move, cpuctl_move);
12456 	CID_OFFSET_MATCH(cgroup_cancel_move, cpuctl_cancel_move);
12457 	CID_OFFSET_MATCH(cgroup_set_weight, cpuctl_set_weight);
12458 	CID_OFFSET_MATCH(cgroup_set_bandwidth, cpuctl_set_bandwidth);
12459 	CID_OFFSET_MATCH(cgroup_set_idle, cpuctl_set_idle);
12460 #endif
12461 	/* @priv tail must align since both share the same data block */
12462 	CID_OFFSET_MATCH(priv, priv);
12463 	/*
12464 	 * cid-form must end exactly at @priv - scx_validate_ops() skips
12465 	 * cpu_acquire/cpu_release for cid-form because reading those fields
12466 	 * past the BPF allocation would be UB.
12467 	 */
12468 	BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) !=
12469 		     offsetofend(struct sched_ext_ops, priv));
12470 #undef CID_OFFSET_MATCH
12471 
12472 	/*
12473 	 * kfunc registration can't be done from init_sched_ext_class() as
12474 	 * register_btf_kfunc_id_set() needs most of the system to be up.
12475 	 *
12476 	 * Some kfuncs are context-sensitive and can only be called from
12477 	 * specific SCX ops. They are grouped into per-context BTF sets, each
12478 	 * registered with scx_kfunc_context_filter as its .filter callback. The
12479 	 * BPF core dedups identical filter pointers per hook
12480 	 * (btf_populate_kfunc_set()), so the filter is invoked exactly once per
12481 	 * kfunc lookup; it consults scx_kf_allow_flags[] to enforce per-op
12482 	 * restrictions at verify time.
12483 	 */
12484 	if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
12485 					     &scx_kfunc_set_enqueue_dispatch)) ||
12486 	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
12487 					     &scx_kfunc_set_dispatch)) ||
12488 	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
12489 					     &scx_kfunc_set_cpu_release)) ||
12490 	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
12491 					     &scx_kfunc_set_unlocked)) ||
12492 	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
12493 					     &scx_kfunc_set_unlocked)) ||
12494 	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
12495 					     &scx_kfunc_set_any)) ||
12496 	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
12497 					     &scx_kfunc_set_any)) ||
12498 	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
12499 					     &scx_kfunc_set_any))) {
12500 		pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret);
12501 		return ret;
12502 	}
12503 
12504 	ret = scx_idle_init();
12505 	if (ret) {
12506 		pr_err("sched_ext: Failed to initialize idle tracking (%d)\n", ret);
12507 		return ret;
12508 	}
12509 
12510 	ret = scx_cid_kfunc_init();
12511 	if (ret) {
12512 		pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret);
12513 		return ret;
12514 	}
12515 
12516 	ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops);
12517 	if (ret) {
12518 		pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret);
12519 		return ret;
12520 	}
12521 
12522 	ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid);
12523 	if (ret) {
12524 		pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret);
12525 		return ret;
12526 	}
12527 
12528 	ret = register_pm_notifier(&scx_pm_notifier);
12529 	if (ret) {
12530 		pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret);
12531 		return ret;
12532 	}
12533 
12534 	scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj);
12535 	if (!scx_kset) {
12536 		pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n");
12537 		return -ENOMEM;
12538 	}
12539 
12540 	ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group);
12541 	if (ret < 0) {
12542 		pr_err("sched_ext: Failed to add global attributes\n");
12543 		return ret;
12544 	}
12545 
12546 	return 0;
12547 }
12548 __initcall(scx_init);
12549 
12550 /*
12551  * Compatibility markers for userspace. Existence of a marker function
12552  * represents that the kernel supports that sched-ext feature.
12553  */
12554 
12555 /*
12556  * scx_compat_marker_cgroup_set_bandwidth_may_sleep: advertises that
12557  * ops.cgroup_set_bandwidth() may be implemented as a sleepable callback.
12558  */
12559 #ifdef CONFIG_EXT_GROUP_SCHED
12560 DEFINE_SCX_COMPAT_MARKER(cgroup_set_bandwidth_may_sleep);
12561 #endif	/* CONFIG_EXT_GROUP_SCHED */
12562